vision-agent 0.2.246__py3-none-any.whl → 1.0.2__py3-none-any.whl

vision_agent/agent/README.md

@@ -33,7 +33,7 @@ The `VisionAgentPlannerV2` returns `PlanContext` which contains the a finalized
 
 
 #### Callbacks
-If you want to recieve intermediate messages you can use the `update_callback` argument in all the `V2` constructors. This will asynchronously send `AgentMessage`'s to the callback function you provide. You can see an example of how to run this in `examples/chat/app.py`
+If you want to receive intermediate messages you can use the `update_callback` argument in all the `V2` constructors. This will asynchronously send `AgentMessage`'s to the callback function you provide. You can see an example of how to run this in `examples/chat/app.py`
 
 ### Human-in-the-loop
 Human-in-the-loop is a feature that allows the user to interact with the agents at certain points in the conversation. This is handled by using the `interaction` and `interaction_response` roles in the `AgentMessage`. You can enable this feature by passing `hil=True` to the `VisionAgentV2`, currently you can only use human-in-the-loop if you are also using the `update_callback` to collect the messages and pass them back to `VisionAgentV2`.

vision_agent/agent/vision_agent_coder_v2.py

@@ -78,7 +78,7 @@ def write_code(
         plan=plan,
     )
     response = cast(str, coder([{"role": "user", "content": prompt}], stream=False))
-    maybe_code = extract_tag(response, "code")
+    maybe_code = extract_tag(response, "code", extract_markdown="python")
 
     # if the response wasn't properly formatted with the code tags just retrun the response
     if maybe_code is None:
@@ -105,7 +105,7 @@ def write_test(
         media=media_list,
     )
     response = cast(str, tester([{"role": "user", "content": prompt}], stream=False))
-    maybe_code = extract_tag(response, "code")
+    maybe_code = extract_tag(response, "code", extract_markdown="python")
 
     # if the response wasn't properly formatted with the code tags just retrun the response
     if maybe_code is None:
@@ -151,8 +151,12 @@ def debug_code(
             fixed_code_and_test_str = cast(str, fixed_code_and_test_str)
             thoughts_tag = extract_tag(fixed_code_and_test_str, "thoughts")
             thoughts = thoughts_tag if thoughts_tag is not None else ""
-            fixed_code = extract_tag(fixed_code_and_test_str, "code")
-            fixed_test = extract_tag(fixed_code_and_test_str, "test")
+            fixed_code = extract_tag(
+                fixed_code_and_test_str, "code", extract_markdown="python"
+            )
+            fixed_test = extract_tag(
+                fixed_code_and_test_str, "test", extract_markdown="python"
+            )
 
             success = not (fixed_code is None and fixed_test is None)
 

vision_agent/agent/vision_agent_planner_v2.py

@@ -154,7 +154,7 @@ def run_multi_trial_planning(
         response3=responses[2],
     )
     response = cast(str, model.chat([{"role": "user", "content": prompt}]))
-    json_str = extract_tag(response, "json")
+    json_str = extract_tag(response, "json", extract_markdown="json")
     if json_str:
         json_data = extract_json(json_str)
         best = np.argmax([int(json_data[f"response{k}"]) for k in [1, 2, 3]])
@@ -230,7 +230,7 @@ def execute_code_action(
     while not execution.success and count <= 3:
         prompt = FIX_BUG.format(chat_history=get_planning(chat), code=code, error=obs)
         response = cast(str, model.chat([{"role": "user", "content": prompt}]))
-        new_code = extract_tag(response, "code")
+        new_code = extract_tag(response, "code", extract_markdown="python")
         if not new_code:
             continue
         else:
@@ -343,7 +343,7 @@ def create_finalize_plan(
     plan_str = cast(str, response)
     return_chat = [AgentMessage(role="planner", content=plan_str, media=None)]
 
-    plan_json = extract_tag(plan_str, "json")
+    plan_json = extract_tag(plan_str, "json", extract_markdown="json")
 
     # sometimes the planner model will refuse to answer a question becuase of some
     # safety concern, we then wont be able to parse the response so we have to send
@@ -357,7 +357,7 @@ def create_finalize_plan(
     except json.JSONDecodeError:
         return return_chat, ErrorContext(error=plan_str)
 
-    code_snippets = extract_tag(plan_str, "code")
+    code_snippets = extract_tag(plan_str, "code", extract_markdown="python")
     plan["code"] = code_snippets if code_snippets is not None else ""
     if verbose:
         _CONSOLE.print(
@@ -544,7 +544,9 @@ class VisionAgentPlannerV2(AgentPlanner):
 
                 response = response_safeguards(response)
                 thinking = extract_tag(response, "thinking")
-                code = extract_tag(response, "execute_python")
+                code = extract_tag(
+                    response, "execute_python", extract_markdown="python"
+                )
                 finalize_plan = extract_tag(response, "finalize_plan")
                 finished = finalize_plan is not None
                 self.update_callback({"role": "planner_update", "content": response})

vision_agent/sim/sim.py

@@ -14,7 +14,7 @@ from scipy.spatial.distance import cosine  # type: ignore
 
 from vision_agent.tools.tools import get_tools_df
 from vision_agent.utils.tools import (
-    _LND_API_KEY,
+    get_vision_agent_api_key,
     _create_requests_session,
     _LND_API_URL_v2,
 )
@@ -57,7 +57,8 @@ def stella_embeddings(prompts: List[str]) -> List[np.ndarray]:
         "model": "stella1.5b",
     }
     url = f"{_LND_API_URL_v2}/embeddings"
-    headers = {"apikey": _LND_API_KEY}
+    vision_agent_api_key = get_vision_agent_api_key()
+    headers = {"Authorization": f"Basic {vision_agent_api_key}"}
     session = _create_requests_session(
         url=url,
         num_retry=3,

vision_agent/tools/planner_tools.py

@@ -130,7 +130,7 @@ def run_multi_judge(
 
     def run_judge() -> Tuple[Optional[Callable], str, str]:
         response = tool_chooser.generate(prompt, media=image_paths, temperature=1.0)
-        tool_choice_context = extract_tag(response, "json")  # type: ignore
+        tool_choice_context = extract_tag(response, "json", extract_markdown="json")  # type: ignore
         tool_choice_context_dict = extract_json(tool_choice_context)  # type: ignore
         tool, tool_thoughts, tool_docstring, _ = extract_tool_info(
             tool_choice_context_dict
@@ -275,7 +275,7 @@ def run_tool_testing(
     )
 
     response = lmm.generate(prompt, media=image_paths)
-    code = extract_tag(response, "code")  # type: ignore
+    code = extract_tag(response, "code", extract_markdown="python")  # type: ignore
     if code is None:
         raise ValueError(f"Could not extract code from response: {response}")
 
@@ -304,7 +304,7 @@ def run_tool_testing(
             media=str(image_paths),
         )
         response = cast(str, lmm.generate(prompt, media=image_paths))
-        code = extract_tag(response, "code")
+        code = extract_tag(response, "code", extract_markdown="python")
         if code is None:
             code = response
 

vision_agent/utils/agent.py

@@ -103,24 +103,37 @@ def extract_code(code: str) -> str:
     return code
 
 
-def extract_tag(
-    content: str,
-    tag: str,
-) -> Optional[str]:
+def _extract_arbitrary(content: str, start: str, stop: str) -> Optional[str]:
     inner_content = None
     remaning = content
     all_inner_content = []
 
-    while f"<{tag}>" in remaning:
-        inner_content_i = remaning[remaning.find(f"<{tag}>") + len(f"<{tag}>") :]
-        if f"</{tag}>" not in inner_content_i:
+    while start in remaning:
+        inner_content_i = remaning[remaning.find(start) + len(start) :]
+        if stop not in inner_content_i:
             break
-        inner_content_i = inner_content_i[: inner_content_i.find(f"</{tag}>")]
-        remaning = remaning[remaning.find(f"</{tag}>") + len(f"</{tag}>") :]
+        inner_content_i = inner_content_i[: inner_content_i.find(stop)]
+        remaning = remaning[remaning.find(stop) + len(stop) :]
         all_inner_content.append(inner_content_i)
 
     if len(all_inner_content) > 0:
         inner_content = "\n".join(all_inner_content)
+
+    return inner_content
+
+
+def extract_markdown(content: str, tag: str) -> Optional[str]:
+    return _extract_arbitrary(content, f"```{tag}", "```")
+
+
+def extract_tag(
+    content: str,
+    tag: str,
+    extract_markdown: Optional[str] = None,
+) -> Optional[str]:
+    inner_content = _extract_arbitrary(content, f"<{tag}>", f"</{tag}>")
+    if inner_content is None and extract_markdown is not None:
+        inner_content = _extract_arbitrary(content, f"```{extract_markdown}", "```")
     return inner_content
 
 

vision_agent/utils/tools.py

@@ -1,6 +1,7 @@
 import logging
 import os
 from base64 import b64encode
+from functools import cache
 from typing import Any, Dict, List, MutableMapping, Optional, Tuple
 
 import numpy as np
@@ -13,15 +14,22 @@ from urllib3.util.retry import Retry
 from vision_agent.utils.exceptions import RemoteToolCallFailed
 from vision_agent.utils.execute import Error, MimeType
 from vision_agent.utils.image_utils import normalize_bbox
-from vision_agent.utils.type_defs import LandingaiAPIKey
 
 _LOGGER = logging.getLogger(__name__)
-_LND_API_KEY = os.environ.get("LANDINGAI_API_KEY", LandingaiAPIKey().api_key)
-_LND_BASE_URL = os.environ.get("LANDINGAI_URL", "https://api.landing.ai")
+_LND_BASE_URL = os.environ.get("LANDINGAI_URL", "https://api.va.landing.ai")
 _LND_API_URL = f"{_LND_BASE_URL}/v1/agent/model"
 _LND_API_URL_v2 = f"{_LND_BASE_URL}/v1/tools"
 
 
+@cache
+def get_vision_agent_api_key() -> str:
+    vision_agent_api_key = os.environ.get("VISION_AGENT_API_KEY")
+    if vision_agent_api_key:
+        return vision_agent_api_key
+    else:
+        raise ValueError("VISION_AGENT_API_KEY not found in environment variables.")
+
+
 def should_report_tool_traces() -> bool:
     return bool(os.environ.get("REPORT_TOOL_TRACES", False))
 
@@ -47,12 +55,13 @@ def send_inference_request(
     if "TOOL_ENDPOINT_URL" in os.environ:
         url = os.environ["TOOL_ENDPOINT_URL"]
 
-    headers = {"apikey": _LND_API_KEY}
+    vision_agent_api_key = get_vision_agent_api_key()
+    headers = {"Authorization": f"Basic {vision_agent_api_key}"}
     if "TOOL_ENDPOINT_AUTH" in os.environ:
         headers["Authorization"] = os.environ["TOOL_ENDPOINT_AUTH"]
         headers.pop("apikey")
 
-    if runtime_tag := os.environ.get("RUNTIME_TAG", ""):
+    if runtime_tag := os.environ.get("RUNTIME_TAG", "vision-agent"):
         headers["runtime_tag"] = runtime_tag
 
     session = _create_requests_session(
@@ -80,7 +89,8 @@ def send_task_inference_request(
     is_form: bool = False,
 ) -> Any:
     url = f"{_LND_API_URL_v2}/{task_name}"
-    headers = {"apikey": _LND_API_KEY}
+    vision_agent_api_key = get_vision_agent_api_key()
+    headers = {"Authorization": f"Basic {vision_agent_api_key}"}
     session = _create_requests_session(
         url=url,
         num_retry=3,

{vision_agent-0.2.246.dist-info → vision_agent-1.0.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.246
+Version: 1.0.2
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai
@@ -11,6 +11,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Requires-Dist: anthropic (>=0.31.0,<0.32.0)
 Requires-Dist: av (>=11.0.0,<12.0.0)
+Requires-Dist: dotenv (>=0.9.9,<0.10.0)
 Requires-Dist: e2b (>=0.17.2a50,<0.18.0)
 Requires-Dist: e2b-code-interpreter (==0.0.11a37)
 Requires-Dist: flake8 (>=7.0.0,<8.0.0)
@@ -27,7 +28,6 @@ Requires-Dist: pandas (>=2.0.0,<3.0.0)
 Requires-Dist: pillow (>=10.0.0,<11.0.0)
 Requires-Dist: pillow-heif (>=0.16.0,<0.17.0)
 Requires-Dist: pydantic (==2.7.4)
-Requires-Dist: pydantic-settings (>=2.2.1,<3.0.0)
 Requires-Dist: pytube (==15.0.0)
 Requires-Dist: requests (>=2.0.0,<3.0.0)
 Requires-Dist: rich (>=13.7.1,<14.0.0)
@@ -72,6 +72,12 @@ export OPENAI_API_KEY="your-api-key"
 
 > **_NOTE:_** We found using both Anthropic Claude-3.5 and OpenAI o1 to be provide the best performance for VisionAgent. If you want to use a different LLM provider or only one, see 'Using Other LLM Providers' below.
 
+You will also need to set your VisionAgent API key to be able to authenticate when using the hosted vision tools that we provide through our APIs. Currently, the APIs are free to use so you will only need to get it from [here](https://va.landing.ai/account/api-key).
+
+```bash
+export VISION_AGENT_API_KEY="your-api-key"
+```
+
 ## Documentation
 
 [VisionAgent Library Docs](https://landing-ai.github.io/vision-agent/)

{vision_agent-0.2.246.dist-info → vision_agent-1.0.2.dist-info}/RECORD

@@ -1,18 +1,18 @@
 vision_agent/.sim_tools/df.csv,sha256=mIr1iubLDqGsL3K3ab6bmh6PtLvmOpvnaIX28lxdV6c,40706
 vision_agent/.sim_tools/embs.npy,sha256=pZZMFMg0rkIAOpMOjN7gjD58hPK07c2ylfQ9YST8xFA,245888
 vision_agent/__init__.py,sha256=EAb4-f9iyuEYkBrX4ag1syM8Syx8118_t0R6_C34M9w,57
-vision_agent/agent/README.md,sha256=Q4w7FWw38qaWosQYAZ7NqWx8Q5XzuWrlv7nLhjUd1-8,5527
+vision_agent/agent/README.md,sha256=3XSPG_VO7-6y6P8COvcgSSonWj5uvfgvfmOkBpfKK8Q,5527
 vision_agent/agent/__init__.py,sha256=M8CffavdIh8Zh-skznLHIaQkYGCGK7vk4dq1FaVkbs4,617
 vision_agent/agent/agent.py,sha256=o1Zuhl6h2R7uVwvUur0Aj38kak8U08plfeFWPst_ErM,1576
 vision_agent/agent/vision_agent.py,sha256=4LqvwPTSsiuJEDwBbMx9Dg9ALJwNR6x1c63TZvOMm8A,23486
 vision_agent/agent/vision_agent_coder.py,sha256=Ry6AiyAj3hsSeYPu_5guMcTzf2E4SoebPzpHyJtSPbQ,27360
 vision_agent/agent/vision_agent_coder_prompts.py,sha256=D4RJxTWoxpl-WtYRvHNxaLSdWVHsdYb0jJIQ2ZCGU0A,12277
 vision_agent/agent/vision_agent_coder_prompts_v2.py,sha256=53b_DhQtffX5wxLuCbNQ83AJhB0P_3wEnuKr-v5bx-o,4866
-vision_agent/agent/vision_agent_coder_v2.py,sha256=I4gWrneFIqhX6W-MxiaNyPKGk5tRKgC8xryV-YdeSZU,17289
+vision_agent/agent/vision_agent_coder_v2.py,sha256=VXLYtBG65gTc1X7QFcicNmUmaj8V6ZIu6yHcIW33TvM,17457
 vision_agent/agent/vision_agent_planner.py,sha256=8LeUsxUlGuQMqCdJ6jHXOe6RpKpzLtMW1AaJlfuIfzw,18680
 vision_agent/agent/vision_agent_planner_prompts.py,sha256=rYRdJthc-sQN57VgCBKrF09Sd73BSxcBdjNe6C4WNZ8,6837
 vision_agent/agent/vision_agent_planner_prompts_v2.py,sha256=TiiF5BGnFVraFlQnDaeRU67927LvszvpcMUOgVgo0ps,35843
-vision_agent/agent/vision_agent_planner_v2.py,sha256=3Rt0kAIpIiufdEuO5-0IpUitou-QlhWXArNXTgoRzxw,21848
+vision_agent/agent/vision_agent_planner_v2.py,sha256=Ot6ugP8pGbD2iNoOwRHKV6-nyJ0LotdEy0uEO-k_jAI,22017
 vision_agent/agent/vision_agent_prompts.py,sha256=KaJwYPUP7_GvQsCPPs6Fdawmi3AQWmWajBUuzj7gTG4,13812
 vision_agent/agent/vision_agent_prompts_v2.py,sha256=OnHmmteAj8__2spEXNcckApQvEpkK_JIZd_SWzEyg9c,4230
 vision_agent/agent/vision_agent_v2.py,sha256=QPAyDjnRRHUCD4Pw4TQYffWkucbn4WkEjYn8dBIWll4,11682
@@ -32,23 +32,22 @@ vision_agent/models/agent_types.py,sha256=vBZ9-ns5lHDdFMO7ulCGGeZ6OwRo3gK4O3vN08
 vision_agent/models/lmm_types.py,sha256=v04h-NjbczHOIN8UWa1vvO5-1BDuZ4JQhD2mge1cXmw,305
 vision_agent/models/tools_types.py,sha256=8hYf2OZhI58gvf65KGaeGkt4EQ56nwLFqIQDPHioOBc,2339
 vision_agent/sim/__init__.py,sha256=Aouz6HEPPTYcLxR5_0fTYCL1OvPKAH1RMWAF90QXAlA,135
-vision_agent/sim/sim.py,sha256=VSU_1rYd4ifvF45xKWBEYugxdeeEQVpj0QL6rjx49i4,9801
+vision_agent/sim/sim.py,sha256=xkioTFNTPl7wAnNgIFfNzswKK_9dRQ6G8OL8-K4CqsI,9893
 vision_agent/tools/__init__.py,sha256=H7FWx0OXGVIjrSOTpNH-YwE4LBuOfThZTG-SHFpo_Z8,2576
 vision_agent/tools/meta_tools.py,sha256=DNRXHX9nZ1GBeqeLiq87sBshoe0aiZeYasETbG-9neI,24053
-vision_agent/tools/planner_tools.py,sha256=orBTdJQz2NKoLuX9WE6XixaYuG305xz0UBYvZOiuquQ,19474
+vision_agent/tools/planner_tools.py,sha256=mlpUODpsN9sg-OKdsBm-6maK3eA97FUgAWm1etfgx4E,19553
 vision_agent/tools/prompts.py,sha256=V1z4YJLXZuUl_iZ5rY0M5hHc_2tmMEUKr0WocXKGt4E,1430
 vision_agent/tools/tools.py,sha256=-jBrykNYPinRpDXnBsnzlSgJ_hbZClzCp3pkzWjTUxs,122098
 vision_agent/utils/__init__.py,sha256=mANUs_84VL-3gpZbXryvV2mWU623eWnRlJCSUHtMjuw,122
-vision_agent/utils/agent.py,sha256=8z4Ei0q397lVWUga8v9nQKuenGAsh2wfkAKQOB8CwpI,14701
+vision_agent/utils/agent.py,sha256=2ifTP5QElItnr4YHOJR6L5P1PUzV0GhChTTqVxuVyQg,15153
 vision_agent/utils/exceptions.py,sha256=zis8smCbdEylBVZBTVfEUfAh7Rb7cWV3MSPambu6FsQ,1837
 vision_agent/utils/execute.py,sha256=vOEP5Ys7S2lc0_7pOJbgk7OaWi85hrCNu9_8Bo3zk6I,29356
 vision_agent/utils/image_utils.py,sha256=bJM2mEvB6E__M9pxi74yQYzAiZ7mu3KE2ptyVrp5vzQ,12533
-vision_agent/utils/tools.py,sha256=USZL0MKsiJgqA8RFiYRTcj_Kn2FVYKLHK4wIk0gP1Ow,7694
+vision_agent/utils/tools.py,sha256=lGELSZ3xmrUa1cHR4ey_f9gyDvWsIA8Q2R7G9LNGKiE,8036
 vision_agent/utils/tools_doc.py,sha256=yFue6KSXoa_Z1ngCdBEc4SdPZOWF1rVLeaHu02I8Wis,2523
-vision_agent/utils/type_defs.py,sha256=BE12s3JNQy36QvauXHjwyeffVh5enfcvd4vTzSwvEZI,1384
 vision_agent/utils/video.py,sha256=rjsQ1sKKisaQ6AVjJz0zd_G4g-ovRweS_rs4JEhenoI,5340
 vision_agent/utils/video_tracking.py,sha256=DZLFpNCuzuPJQzbQoVNcp-m4dKxgiKdCNM5QTh_zURE,12245
-vision_agent-0.2.246.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-vision_agent-0.2.246.dist-info/METADATA,sha256=O0vPLVrDBwISB1hDjMejqtvqTBfE6IaSzBZBVsnTLhE,5712
-vision_agent-0.2.246.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-vision_agent-0.2.246.dist-info/RECORD,,
+vision_agent-1.0.2.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+vision_agent-1.0.2.dist-info/METADATA,sha256=GHEbZmUIjQP9Qg5EHavhbJN9u9kreWF9DGDGBvI8_Y8,6024
+vision_agent-1.0.2.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+vision_agent-1.0.2.dist-info/RECORD,,

vision_agent/utils/type_defs.py

@@ -1,38 +0,0 @@
-from pydantic import Field, field_validator
-from pydantic_settings import BaseSettings
-
-from vision_agent.utils.exceptions import InvalidApiKeyError
-
-
-class LandingaiAPIKey(BaseSettings):
-    """The API key of a user in a particular organization in LandingLens.
-    It supports loading from environment variables or .env files.
-    The supported name of the environment variables are (case-insensitive):
-    - LANDINGAI_API_KEY
-
-    Environment variables will always take priority over values loaded from a dotenv file.
-    """
-
-    api_key: str = Field(
-        default="land_sk_zKvyPcPV2bVoq7q87KwduoerAxuQpx33DnqP8M1BliOCiZOSoI",
-        alias="LANDINGAI_API_KEY",
-        description="The API key of LandingAI.",
-    )
-
-    @field_validator("api_key")
-    @classmethod
-    def is_api_key_valid(cls, key: str) -> str:
-        """Check if the API key is a v2 key."""
-        if not key:
-            raise InvalidApiKeyError(f"LandingAI API key is required, but it's {key}")
-        if not key.startswith("land_sk_"):
-            raise InvalidApiKeyError(
-                f"LandingAI API key (v2) must start with 'land_sk_' prefix, but it's {key}. See https://support.landing.ai/docs/api-key for more information."
-            )
-        return key
-
-    class Config:
-        env_file = ".env"
-        env_prefix = "landingai_"
-        case_sensitive = False
-        extra = "ignore"