vision-agent 0.2.245__py3-none-any.whl → 1.0.1__py3-none-any.whl

vision_agent/agent/vision_agent_coder_v2.py

@@ -78,7 +78,7 @@ def write_code(
         plan=plan,
     )
     response = cast(str, coder([{"role": "user", "content": prompt}], stream=False))
-    maybe_code = extract_tag(response, "code")
+    maybe_code = extract_tag(response, "code", extract_markdown="python")
 
     # if the response wasn't properly formatted with the code tags just retrun the response
     if maybe_code is None:
@@ -105,7 +105,7 @@ def write_test(
         media=media_list,
     )
     response = cast(str, tester([{"role": "user", "content": prompt}], stream=False))
-    maybe_code = extract_tag(response, "code")
+    maybe_code = extract_tag(response, "code", extract_markdown="python")
 
     # if the response wasn't properly formatted with the code tags just retrun the response
     if maybe_code is None:
@@ -151,8 +151,12 @@ def debug_code(
             fixed_code_and_test_str = cast(str, fixed_code_and_test_str)
             thoughts_tag = extract_tag(fixed_code_and_test_str, "thoughts")
             thoughts = thoughts_tag if thoughts_tag is not None else ""
-            fixed_code = extract_tag(fixed_code_and_test_str, "code")
-            fixed_test = extract_tag(fixed_code_and_test_str, "test")
+            fixed_code = extract_tag(
+                fixed_code_and_test_str, "code", extract_markdown="python"
+            )
+            fixed_test = extract_tag(
+                fixed_code_and_test_str, "test", extract_markdown="python"
+            )
 
             success = not (fixed_code is None and fixed_test is None)
 

vision_agent/agent/vision_agent_planner.py

@@ -410,7 +410,7 @@ class VisionAgentPlanner(Agent):
             plans = write_plans(
                 chat,
                 get_tool_descriptions_by_names(
-                    custom_tool_names, T.FUNCTION_TOOLS, T.UTIL_TOOLS  # type: ignore
+                    custom_tool_names, T.tools.FUNCTION_TOOLS, T.tools.UTIL_TOOLS  # type: ignore
                 ),
                 format_feedback(working_memory),
                 self.planner,

vision_agent/agent/vision_agent_planner_v2.py

@@ -154,7 +154,7 @@ def run_multi_trial_planning(
         response3=responses[2],
     )
     response = cast(str, model.chat([{"role": "user", "content": prompt}]))
-    json_str = extract_tag(response, "json")
+    json_str = extract_tag(response, "json", extract_markdown="json")
     if json_str:
         json_data = extract_json(json_str)
         best = np.argmax([int(json_data[f"response{k}"]) for k in [1, 2, 3]])
@@ -230,7 +230,7 @@ def execute_code_action(
     while not execution.success and count <= 3:
         prompt = FIX_BUG.format(chat_history=get_planning(chat), code=code, error=obs)
         response = cast(str, model.chat([{"role": "user", "content": prompt}]))
-        new_code = extract_tag(response, "code")
+        new_code = extract_tag(response, "code", extract_markdown="python")
         if not new_code:
             continue
         else:
@@ -343,7 +343,7 @@ def create_finalize_plan(
     plan_str = cast(str, response)
     return_chat = [AgentMessage(role="planner", content=plan_str, media=None)]
 
-    plan_json = extract_tag(plan_str, "json")
+    plan_json = extract_tag(plan_str, "json", extract_markdown="json")
 
     # sometimes the planner model will refuse to answer a question becuase of some
     # safety concern, we then wont be able to parse the response so we have to send
@@ -357,7 +357,7 @@ def create_finalize_plan(
     except json.JSONDecodeError:
         return return_chat, ErrorContext(error=plan_str)
 
-    code_snippets = extract_tag(plan_str, "code")
+    code_snippets = extract_tag(plan_str, "code", extract_markdown="python")
     plan["code"] = code_snippets if code_snippets is not None else ""
     if verbose:
         _CONSOLE.print(
@@ -544,7 +544,9 @@ class VisionAgentPlannerV2(AgentPlanner):
 
                 response = response_safeguards(response)
                 thinking = extract_tag(response, "thinking")
-                code = extract_tag(response, "execute_python")
+                code = extract_tag(
+                    response, "execute_python", extract_markdown="python"
+                )
                 finalize_plan = extract_tag(response, "finalize_plan")
                 finished = finalize_plan is not None
                 self.update_callback({"role": "planner_update", "content": response})

vision_agent/agent/vision_agent_prompts_v2.py

@@ -54,12 +54,22 @@ AGENT: <response>I am VisionAgent, an agent built by LandingAI, to help users wr
 - Understanding documents
 - Pose estimation
 - Visual question answering for both images and videos
-- Action recognition in videos
+- Activity recognition in videos
 - Image inpainting
 
 How can I help you?</response>
 --- END EXAMPLE2 ---
 
+**Helpful Context**:
+- You are helping the user write vision code that utilizes the `vision-agent` library to solve different vision tasks.
+- The goal is to write code the user can use to solve a visual task and then deploy the code so they can run it repeatedly on that task.
+- If the user wants to run the code, they can `pip install vision-agent` to install the library, copy the code, and run it locally themselves.
+- They can find the vision-agent source code at https://github.com/landing-ai/vision-agent. They can find the documentation for vision-agent at https://landing-ai.github.io/vision-agent/.
+- The code utilizes LandingAI's API to run the different tools, the API is free to use and does not require an API key.
+- The user can run VisionAgent (you) from two places, either on our website at https://va.landing.ai or locally.
+- If the user is running VisionAgent on the website, they can only upload images or videos, nothing else at the moment. They can view files created by VisionAgent by clicking the "Files" button in the upper right part of the screen.
+- If the user is running VisionAgent locally they can ask it to do more things like run over files in a folder. They can view files created by VisionAgent by checking their local file system.
+
 **Conversation**:
 Here is the current conversation so far:
 --- START CONVERSATION ---

vision_agent/sim/sim.py

@@ -14,7 +14,7 @@ from scipy.spatial.distance import cosine  # type: ignore
 
 from vision_agent.tools.tools import get_tools_df
 from vision_agent.utils.tools import (
-    _LND_API_KEY,
+    get_vision_agent_api_key,
     _create_requests_session,
     _LND_API_URL_v2,
 )
@@ -57,7 +57,8 @@ def stella_embeddings(prompts: List[str]) -> List[np.ndarray]:
         "model": "stella1.5b",
     }
     url = f"{_LND_API_URL_v2}/embeddings"
-    headers = {"apikey": _LND_API_KEY}
+    vision_agent_api_key = get_vision_agent_api_key()
+    headers = {"Authorization": f"Basic {vision_agent_api_key}"}
     session = _create_requests_session(
         url=url,
         num_retry=3,

vision_agent/tools/planner_tools.py

@@ -130,7 +130,7 @@ def run_multi_judge(
 
     def run_judge() -> Tuple[Optional[Callable], str, str]:
         response = tool_chooser.generate(prompt, media=image_paths, temperature=1.0)
-        tool_choice_context = extract_tag(response, "json")  # type: ignore
+        tool_choice_context = extract_tag(response, "json", extract_markdown="json")  # type: ignore
         tool_choice_context_dict = extract_json(tool_choice_context)  # type: ignore
         tool, tool_thoughts, tool_docstring, _ = extract_tool_info(
             tool_choice_context_dict
@@ -275,7 +275,7 @@ def run_tool_testing(
     )
 
     response = lmm.generate(prompt, media=image_paths)
-    code = extract_tag(response, "code")  # type: ignore
+    code = extract_tag(response, "code", extract_markdown="python")  # type: ignore
     if code is None:
         raise ValueError(f"Could not extract code from response: {response}")
 
@@ -304,7 +304,7 @@ def run_tool_testing(
             media=str(image_paths),
         )
         response = cast(str, lmm.generate(prompt, media=image_paths))
-        code = extract_tag(response, "code")
+        code = extract_tag(response, "code", extract_markdown="python")
         if code is None:
             code = response
 

vision_agent/utils/agent.py

@@ -103,24 +103,37 @@ def extract_code(code: str) -> str:
     return code
 
 
-def extract_tag(
-    content: str,
-    tag: str,
-) -> Optional[str]:
+def _extract_arbitrary(content: str, start: str, stop: str) -> Optional[str]:
     inner_content = None
     remaning = content
     all_inner_content = []
 
-    while f"<{tag}>" in remaning:
-        inner_content_i = remaning[remaning.find(f"<{tag}>") + len(f"<{tag}>") :]
-        if f"</{tag}>" not in inner_content_i:
+    while start in remaning:
+        inner_content_i = remaning[remaning.find(start) + len(start) :]
+        if stop not in inner_content_i:
             break
-        inner_content_i = inner_content_i[: inner_content_i.find(f"</{tag}>")]
-        remaning = remaning[remaning.find(f"</{tag}>") + len(f"</{tag}>") :]
+        inner_content_i = inner_content_i[: inner_content_i.find(stop)]
+        remaning = remaning[remaning.find(stop) + len(stop) :]
         all_inner_content.append(inner_content_i)
 
     if len(all_inner_content) > 0:
         inner_content = "\n".join(all_inner_content)
+
+    return inner_content
+
+
+def extract_markdown(content: str, tag: str) -> Optional[str]:
+    return _extract_arbitrary(content, f"```{tag}", "```")
+
+
+def extract_tag(
+    content: str,
+    tag: str,
+    extract_markdown: Optional[str] = None,
+) -> Optional[str]:
+    inner_content = _extract_arbitrary(content, f"<{tag}>", f"</{tag}>")
+    if inner_content is None and extract_markdown is not None:
+        inner_content = _extract_arbitrary(content, f"```{extract_markdown}", "```")
     return inner_content
 
 

vision_agent/utils/tools.py

@@ -1,6 +1,7 @@
 import logging
 import os
 from base64 import b64encode
+from functools import cache
 from typing import Any, Dict, List, MutableMapping, Optional, Tuple
 
 import numpy as np
@@ -13,15 +14,22 @@ from urllib3.util.retry import Retry
 from vision_agent.utils.exceptions import RemoteToolCallFailed
 from vision_agent.utils.execute import Error, MimeType
 from vision_agent.utils.image_utils import normalize_bbox
-from vision_agent.utils.type_defs import LandingaiAPIKey
 
 _LOGGER = logging.getLogger(__name__)
-_LND_API_KEY = os.environ.get("LANDINGAI_API_KEY", LandingaiAPIKey().api_key)
-_LND_BASE_URL = os.environ.get("LANDINGAI_URL", "https://api.landing.ai")
+_LND_BASE_URL = os.environ.get("LANDINGAI_URL", "https://api.va.landing.ai")
 _LND_API_URL = f"{_LND_BASE_URL}/v1/agent/model"
 _LND_API_URL_v2 = f"{_LND_BASE_URL}/v1/tools"
 
 
+@cache
+def get_vision_agent_api_key() -> str:
+    vision_agent_api_key = os.environ.get("VISION_AGENT_API_KEY")
+    if vision_agent_api_key:
+        return vision_agent_api_key
+    else:
+        raise ValueError("VISION_AGENT_API_KEY not found in environment variables.")
+
+
 def should_report_tool_traces() -> bool:
     return bool(os.environ.get("REPORT_TOOL_TRACES", False))
 
@@ -47,12 +55,13 @@ def send_inference_request(
     if "TOOL_ENDPOINT_URL" in os.environ:
         url = os.environ["TOOL_ENDPOINT_URL"]
 
-    headers = {"apikey": _LND_API_KEY}
+    vision_agent_api_key = get_vision_agent_api_key()
+    headers = {"Authorization": f"Basic {vision_agent_api_key}"}
     if "TOOL_ENDPOINT_AUTH" in os.environ:
         headers["Authorization"] = os.environ["TOOL_ENDPOINT_AUTH"]
         headers.pop("apikey")
 
-    if runtime_tag := os.environ.get("RUNTIME_TAG", ""):
+    if runtime_tag := os.environ.get("RUNTIME_TAG", "vision-agent"):
         headers["runtime_tag"] = runtime_tag
 
     session = _create_requests_session(
@@ -80,7 +89,8 @@ def send_task_inference_request(
     is_form: bool = False,
 ) -> Any:
     url = f"{_LND_API_URL_v2}/{task_name}"
-    headers = {"apikey": _LND_API_KEY}
+    vision_agent_api_key = get_vision_agent_api_key()
+    headers = {"Authorization": f"Basic {vision_agent_api_key}"}
     session = _create_requests_session(
         url=url,
         num_retry=3,

{vision_agent-0.2.245.dist-info → vision_agent-1.0.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.245
+Version: 1.0.1
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai
@@ -11,6 +11,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Requires-Dist: anthropic (>=0.31.0,<0.32.0)
 Requires-Dist: av (>=11.0.0,<12.0.0)
+Requires-Dist: dotenv (>=0.9.9,<0.10.0)
 Requires-Dist: e2b (>=0.17.2a50,<0.18.0)
 Requires-Dist: e2b-code-interpreter (==0.0.11a37)
 Requires-Dist: flake8 (>=7.0.0,<8.0.0)
@@ -27,7 +28,6 @@ Requires-Dist: pandas (>=2.0.0,<3.0.0)
 Requires-Dist: pillow (>=10.0.0,<11.0.0)
 Requires-Dist: pillow-heif (>=0.16.0,<0.17.0)
 Requires-Dist: pydantic (==2.7.4)
-Requires-Dist: pydantic-settings (>=2.2.1,<3.0.0)
 Requires-Dist: pytube (==15.0.0)
 Requires-Dist: requests (>=2.0.0,<3.0.0)
 Requires-Dist: rich (>=13.7.1,<14.0.0)
@@ -72,6 +72,12 @@ export OPENAI_API_KEY="your-api-key"
 
 > **_NOTE:_** We found using both Anthropic Claude-3.5 and OpenAI o1 to be provide the best performance for VisionAgent. If you want to use a different LLM provider or only one, see 'Using Other LLM Providers' below.
 
+You will also need to set your VisionAgent API key to be able to authenticate when using the hosted vision tools that we provide through our APIs. Currently, the APIs are free to use so you will only need to get it from [here](https://va.landing.ai/account/api-key).
+
+```bash
+export VISION_AGENT_API_KEY="your-api-key"
+```
+
 ## Documentation
 
 [VisionAgent Library Docs](https://landing-ai.github.io/vision-agent/)

{vision_agent-0.2.245.dist-info → vision_agent-1.0.1.dist-info}/RECORD

@@ -8,13 +8,13 @@ vision_agent/agent/vision_agent.py,sha256=4LqvwPTSsiuJEDwBbMx9Dg9ALJwNR6x1c63TZv
 vision_agent/agent/vision_agent_coder.py,sha256=Ry6AiyAj3hsSeYPu_5guMcTzf2E4SoebPzpHyJtSPbQ,27360
 vision_agent/agent/vision_agent_coder_prompts.py,sha256=D4RJxTWoxpl-WtYRvHNxaLSdWVHsdYb0jJIQ2ZCGU0A,12277
 vision_agent/agent/vision_agent_coder_prompts_v2.py,sha256=53b_DhQtffX5wxLuCbNQ83AJhB0P_3wEnuKr-v5bx-o,4866
-vision_agent/agent/vision_agent_coder_v2.py,sha256=I4gWrneFIqhX6W-MxiaNyPKGk5tRKgC8xryV-YdeSZU,17289
-vision_agent/agent/vision_agent_planner.py,sha256=rp_atRMDg35WFXNKOTkjUpGPrpSCsiMhcfZtqK-DIV4,18668
+vision_agent/agent/vision_agent_coder_v2.py,sha256=VXLYtBG65gTc1X7QFcicNmUmaj8V6ZIu6yHcIW33TvM,17457
+vision_agent/agent/vision_agent_planner.py,sha256=8LeUsxUlGuQMqCdJ6jHXOe6RpKpzLtMW1AaJlfuIfzw,18680
 vision_agent/agent/vision_agent_planner_prompts.py,sha256=rYRdJthc-sQN57VgCBKrF09Sd73BSxcBdjNe6C4WNZ8,6837
 vision_agent/agent/vision_agent_planner_prompts_v2.py,sha256=TiiF5BGnFVraFlQnDaeRU67927LvszvpcMUOgVgo0ps,35843
-vision_agent/agent/vision_agent_planner_v2.py,sha256=3Rt0kAIpIiufdEuO5-0IpUitou-QlhWXArNXTgoRzxw,21848
+vision_agent/agent/vision_agent_planner_v2.py,sha256=Ot6ugP8pGbD2iNoOwRHKV6-nyJ0LotdEy0uEO-k_jAI,22017
 vision_agent/agent/vision_agent_prompts.py,sha256=KaJwYPUP7_GvQsCPPs6Fdawmi3AQWmWajBUuzj7gTG4,13812
-vision_agent/agent/vision_agent_prompts_v2.py,sha256=jTfu_heNTBaHj1UNI0XIyyFDgDOjPTPP83vrS-g3A1U,2961
+vision_agent/agent/vision_agent_prompts_v2.py,sha256=OnHmmteAj8__2spEXNcckApQvEpkK_JIZd_SWzEyg9c,4230
 vision_agent/agent/vision_agent_v2.py,sha256=QPAyDjnRRHUCD4Pw4TQYffWkucbn4WkEjYn8dBIWll4,11682
 vision_agent/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vision_agent/clients/http.py,sha256=k883i6M_4nl7zwwHSI-yP5sAgQZIDPM1nrKD6YFJ3Xs,2009
@@ -32,23 +32,22 @@ vision_agent/models/agent_types.py,sha256=vBZ9-ns5lHDdFMO7ulCGGeZ6OwRo3gK4O3vN08
 vision_agent/models/lmm_types.py,sha256=v04h-NjbczHOIN8UWa1vvO5-1BDuZ4JQhD2mge1cXmw,305
 vision_agent/models/tools_types.py,sha256=8hYf2OZhI58gvf65KGaeGkt4EQ56nwLFqIQDPHioOBc,2339
 vision_agent/sim/__init__.py,sha256=Aouz6HEPPTYcLxR5_0fTYCL1OvPKAH1RMWAF90QXAlA,135
-vision_agent/sim/sim.py,sha256=VSU_1rYd4ifvF45xKWBEYugxdeeEQVpj0QL6rjx49i4,9801
+vision_agent/sim/sim.py,sha256=xkioTFNTPl7wAnNgIFfNzswKK_9dRQ6G8OL8-K4CqsI,9893
 vision_agent/tools/__init__.py,sha256=H7FWx0OXGVIjrSOTpNH-YwE4LBuOfThZTG-SHFpo_Z8,2576
 vision_agent/tools/meta_tools.py,sha256=DNRXHX9nZ1GBeqeLiq87sBshoe0aiZeYasETbG-9neI,24053
-vision_agent/tools/planner_tools.py,sha256=orBTdJQz2NKoLuX9WE6XixaYuG305xz0UBYvZOiuquQ,19474
+vision_agent/tools/planner_tools.py,sha256=mlpUODpsN9sg-OKdsBm-6maK3eA97FUgAWm1etfgx4E,19553
 vision_agent/tools/prompts.py,sha256=V1z4YJLXZuUl_iZ5rY0M5hHc_2tmMEUKr0WocXKGt4E,1430
 vision_agent/tools/tools.py,sha256=-jBrykNYPinRpDXnBsnzlSgJ_hbZClzCp3pkzWjTUxs,122098
 vision_agent/utils/__init__.py,sha256=mANUs_84VL-3gpZbXryvV2mWU623eWnRlJCSUHtMjuw,122
-vision_agent/utils/agent.py,sha256=8z4Ei0q397lVWUga8v9nQKuenGAsh2wfkAKQOB8CwpI,14701
+vision_agent/utils/agent.py,sha256=2ifTP5QElItnr4YHOJR6L5P1PUzV0GhChTTqVxuVyQg,15153
 vision_agent/utils/exceptions.py,sha256=zis8smCbdEylBVZBTVfEUfAh7Rb7cWV3MSPambu6FsQ,1837
 vision_agent/utils/execute.py,sha256=vOEP5Ys7S2lc0_7pOJbgk7OaWi85hrCNu9_8Bo3zk6I,29356
 vision_agent/utils/image_utils.py,sha256=bJM2mEvB6E__M9pxi74yQYzAiZ7mu3KE2ptyVrp5vzQ,12533
-vision_agent/utils/tools.py,sha256=USZL0MKsiJgqA8RFiYRTcj_Kn2FVYKLHK4wIk0gP1Ow,7694
+vision_agent/utils/tools.py,sha256=lGELSZ3xmrUa1cHR4ey_f9gyDvWsIA8Q2R7G9LNGKiE,8036
 vision_agent/utils/tools_doc.py,sha256=yFue6KSXoa_Z1ngCdBEc4SdPZOWF1rVLeaHu02I8Wis,2523
-vision_agent/utils/type_defs.py,sha256=BE12s3JNQy36QvauXHjwyeffVh5enfcvd4vTzSwvEZI,1384
 vision_agent/utils/video.py,sha256=rjsQ1sKKisaQ6AVjJz0zd_G4g-ovRweS_rs4JEhenoI,5340
 vision_agent/utils/video_tracking.py,sha256=DZLFpNCuzuPJQzbQoVNcp-m4dKxgiKdCNM5QTh_zURE,12245
-vision_agent-0.2.245.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-vision_agent-0.2.245.dist-info/METADATA,sha256=gvPRMzRk4svQwN9n9bVvn9H-J8Nj-0aKvQ134P75hH0,5712
-vision_agent-0.2.245.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-vision_agent-0.2.245.dist-info/RECORD,,
+vision_agent-1.0.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+vision_agent-1.0.1.dist-info/METADATA,sha256=-BKv2FRgq0nAPEgwvlsPNUALsySfnpN6OUBL_1So8Xo,6024
+vision_agent-1.0.1.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+vision_agent-1.0.1.dist-info/RECORD,,

vision_agent/utils/type_defs.py

@@ -1,38 +0,0 @@
-from pydantic import Field, field_validator
-from pydantic_settings import BaseSettings
-
-from vision_agent.utils.exceptions import InvalidApiKeyError
-
-
-class LandingaiAPIKey(BaseSettings):
-    """The API key of a user in a particular organization in LandingLens.
-    It supports loading from environment variables or .env files.
-    The supported name of the environment variables are (case-insensitive):
-    - LANDINGAI_API_KEY
-
-    Environment variables will always take priority over values loaded from a dotenv file.
-    """
-
-    api_key: str = Field(
-        default="land_sk_zKvyPcPV2bVoq7q87KwduoerAxuQpx33DnqP8M1BliOCiZOSoI",
-        alias="LANDINGAI_API_KEY",
-        description="The API key of LandingAI.",
-    )
-
-    @field_validator("api_key")
-    @classmethod
-    def is_api_key_valid(cls, key: str) -> str:
-        """Check if the API key is a v2 key."""
-        if not key:
-            raise InvalidApiKeyError(f"LandingAI API key is required, but it's {key}")
-        if not key.startswith("land_sk_"):
-            raise InvalidApiKeyError(
-                f"LandingAI API key (v2) must start with 'land_sk_' prefix, but it's {key}. See https://support.landing.ai/docs/api-key for more information."
-            )
-        return key
-
-    class Config:
-        env_file = ".env"
-        env_prefix = "landingai_"
-        case_sensitive = False
-        extra = "ignore"