vision-agent 0.2.85__tar.gz → 0.2.87__tar.gz

{vision_agent-0.2.85 → vision_agent-0.2.87}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.85
+Version: 0.2.87
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai

{vision_agent-0.2.85 → vision_agent-0.2.87}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "vision-agent"
-version = "0.2.85"
+version = "0.2.87"
 description = "Toolset for Vision Agent"
 authors = ["Landing AI <dev@landing.ai>"]
 readme = "README.md"

{vision_agent-0.2.85 → vision_agent-0.2.87}/vision_agent/agent/vision_agent.py

@@ -172,19 +172,25 @@ def write_plans(
 def pick_plan(
     chat: List[Message],
     plans: Dict[str, Any],
-    tool_info: str,
+    tool_infos: Dict[str, str],
     model: LMM,
     code_interpreter: CodeInterpreter,
+    test_multi_plan: bool,
     verbosity: int = 0,
     max_retries: int = 3,
-) -> Tuple[str, str]:
+) -> Tuple[Any, str, str]:
+    if not test_multi_plan:
+        k = list(plans.keys())[0]
+        return plans[k], tool_infos[k], ""
+
+    all_tool_info = tool_infos["all"]
     chat = copy.deepcopy(chat)
     if chat[-1]["role"] != "user":
         raise ValueError("Last chat message must be from the user.")
 
     plan_str = format_plans(plans)
     prompt = TEST_PLANS.format(
-        docstring=tool_info, plans=plan_str, previous_attempts=""
+        docstring=all_tool_info, plans=plan_str, previous_attempts=""
     )
 
     code = extract_code(model(prompt))
@@ -201,7 +207,7 @@ def pick_plan(
     count = 0
     while (not tool_output.success or tool_output_str == "") and count < max_retries:
         prompt = TEST_PLANS.format(
-            docstring=tool_info,
+            docstring=all_tool_info,
             plans=plan_str,
             previous_attempts=PREVIOUS_FAILED.format(
                 code=code, error=tool_output.text()
@@ -237,7 +243,17 @@ def pick_plan(
     best_plan = extract_json(model(chat))
     if verbosity >= 1:
         _LOGGER.info(f"Best plan:\n{best_plan}")
-    return best_plan["best_plan"], tool_output_str
+
+    plan = best_plan["best_plan"]
+    if plan in plans and plan in tool_infos:
+        return plans[plan], tool_infos[plan], tool_output_str
+    else:
+        if verbosity >= 1:
+            _LOGGER.warning(
+                f"Best plan {plan} not found in plans or tool_infos. Using the first plan and tool info."
+            )
+        k = list(plans.keys())[0]
+        return plans[k], tool_infos[k], tool_output_str
 
 
 @traceable
@@ -524,6 +540,13 @@ def retrieve_tools(
         )
     all_tools = "\n\n".join(set(tool_info))
     tool_lists_unique["all"] = all_tools
+    log_progress(
+        {
+            "type": "tools",
+            "status": "completed",
+            "payload": tool_lists[list(plans.keys())[0]],
+        }
+    )
     return tool_lists_unique
 
 
@@ -692,6 +715,14 @@ class VisionAgent(Agent):
                 self.planner,
             )
 
+            self.log_progress(
+                {
+                    "type": "plans",
+                    "status": "completed",
+                    "payload": plans[list(plans.keys())[0]],
+                }
+            )
+
             if self.verbosity >= 1 and test_multi_plan:
                 for p in plans:
                     _LOGGER.info(
@@ -705,47 +736,25 @@ class VisionAgent(Agent):
                 self.verbosity,
             )
 
-            if test_multi_plan:
-                best_plan, tool_output_str = pick_plan(
-                    int_chat,
-                    plans,
-                    tool_infos["all"],
-                    self.coder,
-                    code_interpreter,
-                    verbosity=self.verbosity,
-                )
-            else:
-                best_plan = list(plans.keys())[0]
-                tool_output_str = ""
-
-            if best_plan in plans and best_plan in tool_infos:
-                plan_i = plans[best_plan]
-                tool_info = tool_infos[best_plan]
-            else:
-                if self.verbosity >= 1:
-                    _LOGGER.warning(
-                        f"Best plan {best_plan} not found in plans or tool_infos. Using the first plan and tool info."
-                    )
-                k = list(plans.keys())[0]
-                plan_i = plans[k]
-                tool_info = tool_infos[k]
-
-            self.log_progress(
-                {
-                    "type": "plans",
-                    "status": "completed",
-                    "payload": plan_i,
-                }
+            best_plan, best_tool_info, tool_output_str = pick_plan(
+                int_chat,
+                plans,
+                tool_infos,
+                self.coder,
+                code_interpreter,
+                test_multi_plan,
+                verbosity=self.verbosity,
             )
+
             if self.verbosity >= 1:
                 _LOGGER.info(
-                    f"Picked best plan:\n{tabulate(tabular_data=plan_i, headers='keys', tablefmt='mixed_grid', maxcolwidths=_MAX_TABULATE_COL_WIDTH)}"
+                    f"Picked best plan:\n{tabulate(tabular_data=best_plan, headers='keys', tablefmt='mixed_grid', maxcolwidths=_MAX_TABULATE_COL_WIDTH)}"
                 )
 
             results = write_and_test_code(
                 chat=[{"role": c["role"], "content": c["content"]} for c in int_chat],
-                plan="\n-" + "\n-".join([e["instructions"] for e in plan_i]),
-                tool_info=tool_info,
+                plan="\n-" + "\n-".join([e["instructions"] for e in best_plan]),
+                tool_info=best_tool_info,
                 tool_output=tool_output_str,
                 tool_utils=T.UTILITIES_DOCSTRING,
                 working_memory=working_memory,
@@ -761,7 +770,7 @@ class VisionAgent(Agent):
             code = cast(str, results["code"])
             test = cast(str, results["test"])
             working_memory.extend(results["working_memory"])  # type: ignore
-            plan.append({"code": code, "test": test, "plan": plan_i})
+            plan.append({"code": code, "test": test, "plan": best_plan})
 
             execution_result = cast(Execution, results["test_result"])
             self.log_progress(

vision_agent-0.2.87/vision_agent/tools/tool_utils.py

@@ -0,0 +1,95 @@
+import logging
+import os
+from typing import Any, Dict, MutableMapping, Optional
+
+from IPython.display import display
+from pydantic import BaseModel
+from requests import Session
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+from vision_agent.utils.exceptions import RemoteToolCallFailed
+from vision_agent.utils.execute import Error, MimeType
+from vision_agent.utils.type_defs import LandingaiAPIKey
+
+_LOGGER = logging.getLogger(__name__)
+_LND_API_KEY = LandingaiAPIKey().api_key
+_LND_API_URL = "https://api.staging.landing.ai/v1/agent"
+
+
+class ToolCallTrace(BaseModel):
+    endpoint_url: str
+    request: MutableMapping[str, Any]
+    response: MutableMapping[str, Any]
+    error: Optional[Error]
+
+
+def send_inference_request(
+    payload: Dict[str, Any], endpoint_name: str
+) -> Dict[str, Any]:
+    try:
+        if runtime_tag := os.environ.get("RUNTIME_TAG", ""):
+            payload["runtime_tag"] = runtime_tag
+
+        url = f"{_LND_API_URL}/model/{endpoint_name}"
+        if "TOOL_ENDPOINT_URL" in os.environ:
+            url = os.environ["TOOL_ENDPOINT_URL"]
+
+        tool_call_trace = ToolCallTrace(
+            endpoint_url=url,
+            request=payload,
+            response={},
+            error=None,
+        )
+        headers = {"Content-Type": "application/json", "apikey": _LND_API_KEY}
+        if "TOOL_ENDPOINT_AUTH" in os.environ:
+            headers["Authorization"] = os.environ["TOOL_ENDPOINT_AUTH"]
+            headers.pop("apikey")
+
+        session = _create_requests_session(
+            url=url,
+            num_retry=3,
+            headers=headers,
+        )
+        res = session.post(url, json=payload)
+        if res.status_code != 200:
+            tool_call_trace.error = Error(
+                name="RemoteToolCallFailed",
+                value=f"{res.status_code} - {res.text}",
+                traceback_raw=[],
+            )
+            _LOGGER.error(f"Request failed: {res.status_code} {res.text}")
+            raise RemoteToolCallFailed(payload["tool"], res.status_code, res.text)
+
+        resp = res.json()
+        tool_call_trace.response = resp
+        # TODO: consider making the response schema the same between below two sources
+        return resp if "TOOL_ENDPOINT_AUTH" in os.environ else resp["data"]  # type: ignore
+    finally:
+        trace = tool_call_trace.model_dump()
+        trace["type"] = "tool_call"
+        display({MimeType.APPLICATION_JSON: trace}, raw=True)
+
+
+def _create_requests_session(
+    url: str, num_retry: int, headers: Dict[str, str]
+) -> Session:
+    """Create a requests session with retry"""
+    session = Session()
+    retries = Retry(
+        total=num_retry,
+        backoff_factor=2,
+        raise_on_redirect=True,
+        raise_on_status=False,
+        allowed_methods=["GET", "POST", "PUT"],
+        status_forcelist=[
+            408,  # Request Timeout
+            429,  # Too Many Requests (ie. rate limiter).
+            502,  # Bad Gateway
+            503,  # Service Unavailable (include cloud circuit breaker)
+            504,  # Gateway Timeout
+        ],
+    )
+    session.mount(url, HTTPAdapter(max_retries=retries if num_retry > 0 else 0))
+    session.headers.update(headers)
+    return session

{vision_agent-0.2.85 → vision_agent-0.2.87}/vision_agent/utils/exceptions.py

@@ -13,6 +13,15 @@ For more information, see https://landing-ai.github.io/landingai-python/landinga
         return self.message
 
 
+class RemoteToolCallFailed(Exception):
+    """Exception raised when an error occurs during a tool call."""
+
+    def __init__(self, tool_name: str, status_code: int, message: str):
+        self.message = (
+            f"""Tool call ({tool_name}) failed due to {status_code} - {message}"""
+        )
+
+
 class RemoteSandboxError(Exception):
     """Exception related to remote sandbox."""
 

{vision_agent-0.2.85 → vision_agent-0.2.87}/vision_agent/utils/execute.py

@@ -277,6 +277,17 @@ class Error(BaseModel):
         text = "\n".join(self.traceback_raw)
         return _remove_escape_and_color_codes(text) if return_clean_text else text
 
+    @staticmethod
+    def from_exception(e: Exception) -> "Error":
+        """
+        Creates an Error object from an exception.
+        """
+        return Error(
+            name=e.__class__.__name__,
+            value=str(e),
+            traceback_raw=traceback.format_exception(type(e), e, e.__traceback__),
+        )
+
 
 class Execution(BaseModel):
     """

vision_agent-0.2.85/vision_agent/tools/tool_utils.py

@@ -1,67 +0,0 @@
-import logging
-import os
-from typing import Any, Dict
-
-from requests import Session
-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
-
-from vision_agent.utils.type_defs import LandingaiAPIKey
-
-_LOGGER = logging.getLogger(__name__)
-_LND_API_KEY = LandingaiAPIKey().api_key
-_LND_API_URL = "https://api.staging.landing.ai/v1/agent"
-
-
-def send_inference_request(
-    payload: Dict[str, Any], endpoint_name: str
-) -> Dict[str, Any]:
-    if runtime_tag := os.environ.get("RUNTIME_TAG", ""):
-        payload["runtime_tag"] = runtime_tag
-
-    url = f"{_LND_API_URL}/model/{endpoint_name}"
-    if "TOOL_ENDPOINT_URL" in os.environ:
-        url = os.environ["TOOL_ENDPOINT_URL"]
-
-    headers = {"Content-Type": "application/json", "apikey": _LND_API_KEY}
-    if "TOOL_ENDPOINT_AUTH" in os.environ:
-        headers["Authorization"] = os.environ["TOOL_ENDPOINT_AUTH"]
-        headers.pop("apikey")
-
-    session = _create_requests_session(
-        url=url,
-        num_retry=3,
-        headers=headers,
-    )
-    res = session.post(url, json=payload)
-    if res.status_code != 200:
-        _LOGGER.error(f"Request failed: {res.status_code} {res.text}")
-        raise ValueError(f"Request failed: {res.status_code} {res.text}")
-
-    resp = res.json()
-    # TODO: consider making the response schema the same between below two sources
-    return resp if "TOOL_ENDPOINT_AUTH" in os.environ else resp["data"]  # type: ignore
-
-
-def _create_requests_session(
-    url: str, num_retry: int, headers: Dict[str, str]
-) -> Session:
-    """Create a requests session with retry"""
-    session = Session()
-    retries = Retry(
-        total=num_retry,
-        backoff_factor=2,
-        raise_on_redirect=True,
-        raise_on_status=False,
-        allowed_methods=["GET", "POST", "PUT"],
-        status_forcelist=[
-            408,  # Request Timeout
-            429,  # Too Many Requests (ie. rate limiter).
-            502,  # Bad Gateway
-            503,  # Service Unavailable (include cloud circuit breaker)
-            504,  # Gateway Timeout
-        ],
-    )
-    session.mount(url, HTTPAdapter(max_retries=retries if num_retry > 0 else 0))
-    session.headers.update(headers)
-    return session