vision-agent 0.2.59__tar.gz → 0.2.61__tar.gz

{vision_agent-0.2.59 → vision_agent-0.2.61}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vision-agent
-Version: 0.2.59
+Version: 0.2.61
 Summary: Toolset for Vision Agent
 Author: Landing AI
 Author-email: dev@landing.ai
@@ -9,8 +9,8 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: e2b (>=0.17.0,<0.18.0)
-Requires-Dist: e2b-code-interpreter (>=0.0.7,<0.0.8)
+Requires-Dist: e2b (>=0.17.1,<0.18.0)
+Requires-Dist: e2b-code-interpreter (>=0.0.9,<0.0.10)
 Requires-Dist: ipykernel (>=6.29.4,<7.0.0)
 Requires-Dist: langsmith (>=0.1.58,<0.2.0)
 Requires-Dist: moviepy (>=1.0.0,<2.0.0)
@@ -218,13 +218,26 @@ ensure the documentation is in the same format above with description, `Paramete
 `Returns:`, and `Example\n-------`. You can find an example use case [here](examples/custom_tools/).
 
 ### Azure Setup
-If you want to use Azure OpenAI models, you can set the environment variable:
+If you want to use Azure OpenAI models, you need to have two OpenAI model deployments:
+
+1. OpenAI GPT-4o model
+2. OpenAI text embedding model
+
+<img width="1201" alt="Screenshot 2024-06-12 at 5 54 48 PM" src="https://github.com/landing-ai/vision-agent/assets/2736300/da125592-b01d-45bc-bc99-d48c9dcdfa32">
+
+Then you can set the following environment variables:
 
 ```bash
 export AZURE_OPENAI_API_KEY="your-api-key"
 export AZURE_OPENAI_ENDPOINT="your-endpoint"
+# The deployment name of your Azure OpenAI chat model
+export AZURE_OPENAI_CHAT_MODEL_DEPLOYMENT_NAME="your_gpt4o_model_deployment_name"
+# The deployment name of your Azure OpenAI text embedding model
+export AZURE_OPENAI_EMBEDDING_MODEL_DEPLOYMENT_NAME="your_embedding_model_deployment_name"
 ```
 
+> NOTE: make sure your Azure model deployment have enough quota (token per minute) to support it. The default value 8000TPM is not enough.
+
 You can then run Vision Agent using the Azure OpenAI models:
 
 ```python

{vision_agent-0.2.59 → vision_agent-0.2.61}/README.md

@@ -182,13 +182,26 @@ ensure the documentation is in the same format above with description, `Paramete
 `Returns:`, and `Example\n-------`. You can find an example use case [here](examples/custom_tools/).
 
 ### Azure Setup
-If you want to use Azure OpenAI models, you can set the environment variable:
+If you want to use Azure OpenAI models, you need to have two OpenAI model deployments:
+
+1. OpenAI GPT-4o model
+2. OpenAI text embedding model
+
+<img width="1201" alt="Screenshot 2024-06-12 at 5 54 48 PM" src="https://github.com/landing-ai/vision-agent/assets/2736300/da125592-b01d-45bc-bc99-d48c9dcdfa32">
+
+Then you can set the following environment variables:
 
 ```bash
 export AZURE_OPENAI_API_KEY="your-api-key"
 export AZURE_OPENAI_ENDPOINT="your-endpoint"
+# The deployment name of your Azure OpenAI chat model
+export AZURE_OPENAI_CHAT_MODEL_DEPLOYMENT_NAME="your_gpt4o_model_deployment_name"
+# The deployment name of your Azure OpenAI text embedding model
+export AZURE_OPENAI_EMBEDDING_MODEL_DEPLOYMENT_NAME="your_embedding_model_deployment_name"
 ```
 
+> NOTE: make sure your Azure model deployment have enough quota (token per minute) to support it. The default value 8000TPM is not enough.
+
 You can then run Vision Agent using the Azure OpenAI models:
 
 ```python

{vision_agent-0.2.59 → vision_agent-0.2.61}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "vision-agent"
-version = "0.2.59"
+version = "0.2.61"
 description = "Toolset for Vision Agent"
 authors = ["Landing AI <dev@landing.ai>"]
 readme = "README.md"
@@ -34,8 +34,8 @@ nbformat = "^5.10.4"
 rich = "^13.7.1"
 langsmith = "^0.1.58"
 ipykernel = "^6.29.4"
-e2b = "^0.17.0"
-e2b-code-interpreter = "^0.0.7"
+e2b = "^0.17.1"
+e2b-code-interpreter = "^0.0.9"
 tenacity = "^8.3.0"
 
 [tool.poetry.group.dev.dependencies]

{vision_agent-0.2.59 → vision_agent-0.2.61}/vision_agent/agent/vision_agent.py

@@ -8,6 +8,7 @@ from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional, Sequence, Union, cast
 
 from PIL import Image
+from langsmith import traceable
 from rich.console import Console
 from rich.style import Style
 from rich.syntax import Syntax
@@ -130,6 +131,7 @@ def extract_image(
     return new_media
 
 
+@traceable
 def write_plan(
     chat: List[Message],
     tool_desc: str,
@@ -147,6 +149,7 @@ def write_plan(
     return extract_json(model.chat(chat))["plan"]  # type: ignore
 
 
+@traceable
 def write_code(
     coder: LMM,
     chat: List[Message],
@@ -167,6 +170,7 @@ def write_code(
     return extract_code(coder(chat))
 
 
+@traceable
 def write_test(
     tester: LMM,
     chat: List[Message],
@@ -191,6 +195,7 @@ def write_test(
     return extract_code(tester(chat))
 
 
+@traceable
 def reflect(
     chat: List[Message],
     plan: str,
@@ -266,70 +271,19 @@ def write_and_test_code(
     count = 0
     new_working_memory: List[Dict[str, str]] = []
     while not result.success and count < max_retries:
-        log_progress(
-            {
-                "type": "code",
-                "status": "started",
-            }
-        )
-        fixed_code_and_test = extract_json(
-            debugger(
-                FIX_BUG.format(
-                    code=code,
-                    tests=test,
-                    result="\n".join(result.text().splitlines()[-50:]),
-                    feedback=format_memory(working_memory + new_working_memory),
-                )
-            )
-        )
-        old_code = code
-        old_test = test
-
-        if fixed_code_and_test["code"].strip() != "":
-            code = extract_code(fixed_code_and_test["code"])
-        if fixed_code_and_test["test"].strip() != "":
-            test = extract_code(fixed_code_and_test["test"])
-
-        new_working_memory.append(
-            {
-                "code": f"{code}\n{test}",
-                "feedback": fixed_code_and_test["reflections"],
-                "edits": get_diff(f"{old_code}\n{old_test}", f"{code}\n{test}"),
-            }
-        )
-        log_progress(
-            {
-                "type": "code",
-                "status": "running",
-                "payload": {
-                    "code": DefaultImports.prepend_imports(code),
-                    "test": test,
-                },
-            }
-        )
-
-        result = code_interpreter.exec_isolation(
-            f"{DefaultImports.to_code_string()}\n{code}\n{test}"
-        )
-        log_progress(
-            {
-                "type": "code",
-                "status": "completed" if result.success else "failed",
-                "payload": {
-                    "code": DefaultImports.prepend_imports(code),
-                    "test": test,
-                    "result": result.to_json(),
-                },
-            }
-        )
         if verbosity == 2:
-            _LOGGER.info(
-                f"Debug attempt {count + 1}, reflection: {fixed_code_and_test['reflections']}"
-            )
-            _print_code("Code and test after attempted fix:", code, test)
-            _LOGGER.info(
-                f"Code execution result after attempted fix: {result.text(include_logs=True)}"
-            )
+            _LOGGER.info(f"Start debugging attempt {count + 1}")
+        code, test, result = debug_code(
+            working_memory,
+            debugger,
+            code_interpreter,
+            code,
+            test,
+            result,
+            new_working_memory,
+            log_progress,
+            verbosity,
+        )
         count += 1
 
     if verbosity >= 1:
@@ -344,6 +298,83 @@ def write_and_test_code(
     }
 
 
+@traceable
+def debug_code(
+    working_memory: List[Dict[str, str]],
+    debugger: LMM,
+    code_interpreter: CodeInterpreter,
+    code: str,
+    test: str,
+    result: Execution,
+    new_working_memory: List[Dict[str, str]],
+    log_progress: Callable[[Dict[str, Any]], None],
+    verbosity: int = 0,
+) -> tuple[str, str, Execution]:
+    log_progress(
+        {
+            "type": "code",
+            "status": "started",
+        }
+    )
+    fixed_code_and_test = extract_json(
+        debugger(
+            FIX_BUG.format(
+                code=code,
+                tests=test,
+                result="\n".join(result.text().splitlines()[-50:]),
+                feedback=format_memory(working_memory + new_working_memory),
+            )
+        )
+    )
+    old_code = code
+    old_test = test
+
+    if fixed_code_and_test["code"].strip() != "":
+        code = extract_code(fixed_code_and_test["code"])
+    if fixed_code_and_test["test"].strip() != "":
+        test = extract_code(fixed_code_and_test["test"])
+
+    new_working_memory.append(
+        {
+            "code": f"{code}\n{test}",
+            "feedback": fixed_code_and_test["reflections"],
+            "edits": get_diff(f"{old_code}\n{old_test}", f"{code}\n{test}"),
+        }
+    )
+    log_progress(
+        {
+            "type": "code",
+            "status": "running",
+            "payload": {
+                "code": DefaultImports.prepend_imports(code),
+                "test": test,
+            },
+        }
+    )
+
+    result = code_interpreter.exec_isolation(
+        f"{DefaultImports.to_code_string()}\n{code}\n{test}"
+    )
+    log_progress(
+        {
+            "type": "code",
+            "status": "completed" if result.success else "failed",
+            "payload": {
+                "code": DefaultImports.prepend_imports(code),
+                "test": test,
+                "result": result.to_json(),
+            },
+        }
+    )
+    if verbosity == 2:
+        _print_code("Code and test after attempted fix:", code, test)
+        _LOGGER.info(
+            f"Reflection: {fixed_code_and_test['reflections']}\nCode execution result after attempted fix: {result.text(include_logs=True)}"
+        )
+
+    return code, test, result
+
+
 def _print_code(title: str, code: str, test: Optional[str] = None) -> None:
     _CONSOLE.print(title, style=Style(bgcolor="dark_orange3", bold=True))
     _CONSOLE.print("=" * 30 + " Code " + "=" * 30)
@@ -481,6 +512,7 @@ class VisionAgent(Agent):
         results.pop("working_memory")
         return results  # type: ignore
 
+    @traceable
     def chat_with_workflow(
         self,
         chat: List[Message],

{vision_agent-0.2.59 → vision_agent-0.2.61}/vision_agent/lmm/lmm.py

@@ -233,7 +233,7 @@ class OpenAILMM(LMM):
 class AzureOpenAILMM(OpenAILMM):
     def __init__(
         self,
-        model_name: str = "gpt-4o",
+        model_name: Optional[str] = None,
         api_key: Optional[str] = None,
         api_version: str = "2024-02-01",
         azure_endpoint: Optional[str] = None,
@@ -245,14 +245,20 @@ class AzureOpenAILMM(OpenAILMM):
             api_key = os.getenv("AZURE_OPENAI_API_KEY")
         if not azure_endpoint:
             azure_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT")
+        if not model_name:
+            model_name = os.getenv("AZURE_OPENAI_CHAT_MODEL_DEPLOYMENT_NAME")
 
         if not api_key:
             raise ValueError("OpenAI API key is required.")
         if not azure_endpoint:
             raise ValueError("Azure OpenAI endpoint is required.")
+        if not model_name:
+            raise ValueError("Azure OpenAI chat model deployment name is required.")
 
         self.client = AzureOpenAI(
-            api_key=api_key, api_version=api_version, azure_endpoint=azure_endpoint
+            api_key=api_key,
+            api_version=api_version,
+            azure_endpoint=azure_endpoint,
         )
         self.model_name = model_name
 

{vision_agent-0.2.59 → vision_agent-0.2.61}/vision_agent/utils/sim.py

@@ -87,17 +87,23 @@ class AzureSim(Sim):
         api_key: Optional[str] = None,
         api_version: str = "2024-02-01",
         azure_endpoint: Optional[str] = None,
-        model: str = "text-embedding-3-small",
+        model: Optional[str] = None,
     ) -> None:
         if not api_key:
             api_key = os.getenv("AZURE_OPENAI_API_KEY")
         if not azure_endpoint:
             azure_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT")
+        if not model:
+            model = os.getenv("AZURE_OPENAI_EMBEDDING_MODEL_DEPLOYMENT_NAME")
 
         if not api_key:
             raise ValueError("Azure OpenAI API key is required.")
         if not azure_endpoint:
             raise ValueError("Azure OpenAI endpoint is required.")
+        if not model:
+            raise ValueError(
+                "Azure OpenAI embedding model deployment name is required."
+            )
 
         self.df = df
         self.client = AzureOpenAI(