vision-agent 0.2.85__py3-none-any.whl → 0.2.87__py3-none-any.whl

vision_agent/agent/vision_agent.py

@@ -172,19 +172,25 @@ def write_plans(
 def pick_plan(
     chat: List[Message],
     plans: Dict[str, Any],
-    tool_info: str,
+    tool_infos: Dict[str, str],
     model: LMM,
     code_interpreter: CodeInterpreter,
+    test_multi_plan: bool,
     verbosity: int = 0,
     max_retries: int = 3,
-) -> Tuple[str, str]:
+) -> Tuple[Any, str, str]:
+    if not test_multi_plan:
+        k = list(plans.keys())[0]
+        return plans[k], tool_infos[k], ""
+
+    all_tool_info = tool_infos["all"]
     chat = copy.deepcopy(chat)
     if chat[-1]["role"] != "user":
         raise ValueError("Last chat message must be from the user.")
 
     plan_str = format_plans(plans)
     prompt = TEST_PLANS.format(
-        docstring=tool_info, plans=plan_str, previous_attempts=""
+        docstring=all_tool_info, plans=plan_str, previous_attempts=""
     )
 
     code = extract_code(model(prompt))
@@ -201,7 +207,7 @@ def pick_plan(
     count = 0
     while (not tool_output.success or tool_output_str == "") and count < max_retries:
         prompt = TEST_PLANS.format(
-            docstring=tool_info,
+            docstring=all_tool_info,
             plans=plan_str,
             previous_attempts=PREVIOUS_FAILED.format(
                 code=code, error=tool_output.text()
@@ -237,7 +243,17 @@ def pick_plan(
     best_plan = extract_json(model(chat))
     if verbosity >= 1:
         _LOGGER.info(f"Best plan:\n{best_plan}")
-    return best_plan["best_plan"], tool_output_str
+
+    plan = best_plan["best_plan"]
+    if plan in plans and plan in tool_infos:
+        return plans[plan], tool_infos[plan], tool_output_str
+    else:
+        if verbosity >= 1:
+            _LOGGER.warning(
+                f"Best plan {plan} not found in plans or tool_infos. Using the first plan and tool info."
+            )
+        k = list(plans.keys())[0]
+        return plans[k], tool_infos[k], tool_output_str
 
 
 @traceable
@@ -524,6 +540,13 @@ def retrieve_tools(
         )
     all_tools = "\n\n".join(set(tool_info))
     tool_lists_unique["all"] = all_tools
+    log_progress(
+        {
+            "type": "tools",
+            "status": "completed",
+            "payload": tool_lists[list(plans.keys())[0]],
+        }
+    )
     return tool_lists_unique
 
 
@@ -692,6 +715,14 @@ class VisionAgent(Agent):
                 self.planner,
             )
 
+            self.log_progress(
+                {
+                    "type": "plans",
+                    "status": "completed",
+                    "payload": plans[list(plans.keys())[0]],
+                }
+            )
+
             if self.verbosity >= 1 and test_multi_plan:
                 for p in plans:
                     _LOGGER.info(
@@ -705,47 +736,25 @@ class VisionAgent(Agent):
                 self.verbosity,
             )
 
-            if test_multi_plan:
-                best_plan, tool_output_str = pick_plan(
-                    int_chat,
-                    plans,
-                    tool_infos["all"],
-                    self.coder,
-                    code_interpreter,
-                    verbosity=self.verbosity,
-                )
-            else:
-                best_plan = list(plans.keys())[0]
-                tool_output_str = ""
-
-            if best_plan in plans and best_plan in tool_infos:
-                plan_i = plans[best_plan]
-                tool_info = tool_infos[best_plan]
-            else:
-                if self.verbosity >= 1:
-                    _LOGGER.warning(
-                        f"Best plan {best_plan} not found in plans or tool_infos. Using the first plan and tool info."
-                    )
-                k = list(plans.keys())[0]
-                plan_i = plans[k]
-                tool_info = tool_infos[k]
-
-            self.log_progress(
-                {
-                    "type": "plans",
-                    "status": "completed",
-                    "payload": plan_i,
-                }
+            best_plan, best_tool_info, tool_output_str = pick_plan(
+                int_chat,
+                plans,
+                tool_infos,
+                self.coder,
+                code_interpreter,
+                test_multi_plan,
+                verbosity=self.verbosity,
             )
+
             if self.verbosity >= 1:
                 _LOGGER.info(
-                    f"Picked best plan:\n{tabulate(tabular_data=plan_i, headers='keys', tablefmt='mixed_grid', maxcolwidths=_MAX_TABULATE_COL_WIDTH)}"
+                    f"Picked best plan:\n{tabulate(tabular_data=best_plan, headers='keys', tablefmt='mixed_grid', maxcolwidths=_MAX_TABULATE_COL_WIDTH)}"
                 )
 
             results = write_and_test_code(
                 chat=[{"role": c["role"], "content": c["content"]} for c in int_chat],
-                plan="\n-" + "\n-".join([e["instructions"] for e in plan_i]),
-                tool_info=tool_info,
+                plan="\n-" + "\n-".join([e["instructions"] for e in best_plan]),
+                tool_info=best_tool_info,
                 tool_output=tool_output_str,
                 tool_utils=T.UTILITIES_DOCSTRING,
                 working_memory=working_memory,
@@ -761,7 +770,7 @@ class VisionAgent(Agent):
             code = cast(str, results["code"])
             test = cast(str, results["test"])
             working_memory.extend(results["working_memory"])  # type: ignore
-            plan.append({"code": code, "test": test, "plan": plan_i})
+            plan.append({"code": code, "test": test, "plan": best_plan})
 
             execution_result = cast(Execution, results["test_result"])
             self.log_progress(

vision_agent/tools/tool_utils.py

@@ -1,11 +1,15 @@
 import logging
 import os
-from typing import Any, Dict
+from typing import Any, Dict, MutableMapping, Optional
 
+from IPython.display import display
+from pydantic import BaseModel
 from requests import Session
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
+from vision_agent.utils.exceptions import RemoteToolCallFailed
+from vision_agent.utils.execute import Error, MimeType
 from vision_agent.utils.type_defs import LandingaiAPIKey
 
 _LOGGER = logging.getLogger(__name__)
@@ -13,34 +17,58 @@ _LND_API_KEY = LandingaiAPIKey().api_key
 _LND_API_URL = "https://api.staging.landing.ai/v1/agent"
 
 
+class ToolCallTrace(BaseModel):
+    endpoint_url: str
+    request: MutableMapping[str, Any]
+    response: MutableMapping[str, Any]
+    error: Optional[Error]
+
+
 def send_inference_request(
     payload: Dict[str, Any], endpoint_name: str
 ) -> Dict[str, Any]:
-    if runtime_tag := os.environ.get("RUNTIME_TAG", ""):
-        payload["runtime_tag"] = runtime_tag
+    try:
+        if runtime_tag := os.environ.get("RUNTIME_TAG", ""):
+            payload["runtime_tag"] = runtime_tag
 
-    url = f"{_LND_API_URL}/model/{endpoint_name}"
-    if "TOOL_ENDPOINT_URL" in os.environ:
-        url = os.environ["TOOL_ENDPOINT_URL"]
+        url = f"{_LND_API_URL}/model/{endpoint_name}"
+        if "TOOL_ENDPOINT_URL" in os.environ:
+            url = os.environ["TOOL_ENDPOINT_URL"]
 
-    headers = {"Content-Type": "application/json", "apikey": _LND_API_KEY}
-    if "TOOL_ENDPOINT_AUTH" in os.environ:
-        headers["Authorization"] = os.environ["TOOL_ENDPOINT_AUTH"]
-        headers.pop("apikey")
+        tool_call_trace = ToolCallTrace(
+            endpoint_url=url,
+            request=payload,
+            response={},
+            error=None,
+        )
+        headers = {"Content-Type": "application/json", "apikey": _LND_API_KEY}
+        if "TOOL_ENDPOINT_AUTH" in os.environ:
+            headers["Authorization"] = os.environ["TOOL_ENDPOINT_AUTH"]
+            headers.pop("apikey")
 
-    session = _create_requests_session(
-        url=url,
-        num_retry=3,
-        headers=headers,
-    )
-    res = session.post(url, json=payload)
-    if res.status_code != 200:
-        _LOGGER.error(f"Request failed: {res.status_code} {res.text}")
-        raise ValueError(f"Request failed: {res.status_code} {res.text}")
+        session = _create_requests_session(
+            url=url,
+            num_retry=3,
+            headers=headers,
+        )
+        res = session.post(url, json=payload)
+        if res.status_code != 200:
+            tool_call_trace.error = Error(
+                name="RemoteToolCallFailed",
+                value=f"{res.status_code} - {res.text}",
+                traceback_raw=[],
+            )
+            _LOGGER.error(f"Request failed: {res.status_code} {res.text}")
+            raise RemoteToolCallFailed(payload["tool"], res.status_code, res.text)
 
-    resp = res.json()
-    # TODO: consider making the response schema the same between below two sources
-    return resp if "TOOL_ENDPOINT_AUTH" in os.environ else resp["data"]  # type: ignore
+        resp = res.json()
+        tool_call_trace.response = resp
+        # TODO: consider making the response schema the same between below two sources
+        return resp if "TOOL_ENDPOINT_AUTH" in os.environ else resp["data"]  # type: ignore
+    finally:
+        trace = tool_call_trace.model_dump()
+        trace["type"] = "tool_call"
+        display({MimeType.APPLICATION_JSON: trace}, raw=True)
 
 
 def _create_requests_session(

vision_agent/utils/exceptions.py

@@ -13,6 +13,15 @@ For more information, see https://landing-ai.github.io/landingai-python/landinga
         return self.message
 
 
+class RemoteToolCallFailed(Exception):
+    """Exception raised when an error occurs during a tool call."""
+
+    def __init__(self, tool_name: str, status_code: int, message: str):
+        self.message = (
+            f"""Tool call ({tool_name}) failed due to {status_code} - {message}"""
+        )
+
+
 class RemoteSandboxError(Exception):
     """Exception related to remote sandbox."""
 

vision_agent/utils/execute.py

@@ -277,6 +277,17 @@ class Error(BaseModel):
         text = "\n".join(self.traceback_raw)
         return _remove_escape_and_color_codes(text) if return_clean_text else text
 
+    @staticmethod
+    def from_exception(e: Exception) -> "Error":
+        """
+        Creates an Error object from an exception.
+        """
+        return Error(
+            name=e.__class__.__name__,
+            value=str(e),
+            traceback_raw=traceback.format_exception(type(e), e, e.__traceback__),
+        )
+
 
 class Execution(BaseModel):
     """

{vision_agent-0.2.85.dist-info → vision_agent-0.2.87.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.85
+Version: 0.2.87
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai

{vision_agent-0.2.85.dist-info → vision_agent-0.2.87.dist-info}/RECORD

@@ -1,7 +1,7 @@
 vision_agent/__init__.py,sha256=EAb4-f9iyuEYkBrX4ag1syM8Syx8118_t0R6_C34M9w,57
 vision_agent/agent/__init__.py,sha256=IUwfbPMcT8X_rnXMLmI8gJ4ltsHy_XSs9eLiKURJxeY,81
 vision_agent/agent/agent.py,sha256=ZK-5lOtd9-eD9aWcXssJpnOyvZuO7_5hAmnb-6sWVe8,569
-vision_agent/agent/vision_agent.py,sha256=X5V_xBIc672mwnHD20wbz6jKpmfPUwXOooID2moWahM,28996
+vision_agent/agent/vision_agent.py,sha256=MVZmwIk7U7PMwzyqKwhqAI-8Lw1E-X_PdSK9vDmbxxk,29125
 vision_agent/agent/vision_agent_prompts.py,sha256=brBV-SmzyzTG5M9nfV3R5xdYT_BUYOKzxNFmTa2Sp-o,11049
 vision_agent/fonts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vision_agent/fonts/default_font_ch_en.ttf,sha256=1YM0Z3XqLDjSNbF7ihQFSAIUdjF9m1rtHiNC_6QosTE,1594400
@@ -9,16 +9,16 @@ vision_agent/lmm/__init__.py,sha256=j9mQsIXQOYfW6nFd47uTwuBe1ranpEbwW308qLfCWN0,
 vision_agent/lmm/lmm.py,sha256=035uONyp6_jD3PVdNdSg2PMHOG1voqnpsn2IyybUENs,15147
 vision_agent/tools/__init__.py,sha256=k69hvcy2FWjDqVA0klzybKeoToOH_bom5NTVSliA0Og,1838
 vision_agent/tools/prompts.py,sha256=V1z4YJLXZuUl_iZ5rY0M5hHc_2tmMEUKr0WocXKGt4E,1430
-vision_agent/tools/tool_utils.py,sha256=6z0jrvUnesJEFqDHZoAvbXPic8rzh0KfILL07tu0uRo,2205
+vision_agent/tools/tool_utils.py,sha256=ZnqaflVbLZB0GmgJJoQsZZs8hWbODXEPH1_Mq1s4bnc,3222
 vision_agent/tools/tools.py,sha256=TkZqNYX-ocwdaCdXd6c6tysSa_HX2y6Nrgl4JKni4IQ,43661
 vision_agent/utils/__init__.py,sha256=CW84HnhqI6XQVuxf2KifkLnSuO7EOhmuL09-gAymAak,219
-vision_agent/utils/exceptions.py,sha256=JGiFPLAYnPpEvHfueQuJotv3rCkS-A7UAwEym56MmHM,1359
-vision_agent/utils/execute.py,sha256=BRG_HranjVLfDzfiMCPzbH061No2andw4nUVujaMmak,23600
+vision_agent/utils/exceptions.py,sha256=isVH-SVL4vHj3q5kK4z7cy5_aOapAqHXWkpibfSNbUs,1659
+vision_agent/utils/execute.py,sha256=DxuAoKmKAovgKe8IPkwg1B34osoz9_Ouvl1mi8aPXgE,23923
 vision_agent/utils/image_utils.py,sha256=_cdiS5YrLzqkq_ZgFUO897m5M4_SCIThwUy4lOklfB8,7700
 vision_agent/utils/sim.py,sha256=1HTaiVaBiKeyXIy21IYGXlPw0TipOyw9FPOJDfyLI94,4409
 vision_agent/utils/type_defs.py,sha256=QeQRRIlklZMWzxROcCn5ELxP89nYdXGydy1rAiSpZZw,1384
 vision_agent/utils/video.py,sha256=rNmU9KEIkZB5-EztZNlUiKYN0mm_55A_2VGUM0QpqLA,8779
-vision_agent-0.2.85.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-vision_agent-0.2.85.dist-info/METADATA,sha256=AsX-YRr-MS8JvS4tL_KTBS-Bl-RvVUtef9yfqZlvKfk,9477
-vision_agent-0.2.85.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-vision_agent-0.2.85.dist-info/RECORD,,
+vision_agent-0.2.87.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+vision_agent-0.2.87.dist-info/METADATA,sha256=St27tw1lvdjylYwUHeM3928tAzGeJjCj879nMJA-OWw,9477
+vision_agent-0.2.87.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+vision_agent-0.2.87.dist-info/RECORD,,