vision-agent 0.2.228__py3-none-any.whl → 0.2.230__py3-none-any.whl

vision_agent/agent/vision_agent_v2.py

@@ -18,30 +18,15 @@ from vision_agent.agent.types import (
 )
 from vision_agent.agent.vision_agent_coder_v2 import format_code_context
 from vision_agent.agent.vision_agent_prompts_v2 import CONVERSATION
-from vision_agent.lmm import LMM, AnthropicLMM
+from vision_agent.configs import Config
+from vision_agent.lmm import LMM
 from vision_agent.lmm.types import Message
 from vision_agent.utils.execute import CodeInterpreter, CodeInterpreterFactory
 
-
-def run_conversation(agent: LMM, chat: List[AgentMessage]) -> str:
-    # only keep last 10 messages
-    conv = format_conversation(chat[-10:])
-    prompt = CONVERSATION.format(
-        conversation=conv,
-    )
-    response = agent([{"role": "user", "content": prompt}], stream=False)
-    return cast(str, response)
+CONFIG = Config()
 
 
-def check_for_interaction(chat: List[AgentMessage]) -> bool:
-    return (
-        len(chat) > 2
-        and chat[-2].role == "interaction"
-        and chat[-1].role == "interaction_response"
-    )
-
-
-def extract_conversation_for_generate_code(
+def extract_conversation(
     chat: List[AgentMessage],
 ) -> Tuple[List[AgentMessage], Optional[str]]:
     chat = copy.deepcopy(chat)
@@ -75,7 +60,27 @@ def extract_conversation_for_generate_code(
 
         extracted_chat_strip_code = [chat_i] + extracted_chat_strip_code
 
-    return extracted_chat_strip_code[-5:], final_code
+    return extracted_chat_strip_code, final_code
+
+
+def run_conversation(agent: LMM, chat: List[AgentMessage]) -> str:
+    extracted_chat, _ = extract_conversation(chat)
+    extracted_chat = extracted_chat[-10:]
+
+    conv = format_conversation(chat)
+    prompt = CONVERSATION.format(
+        conversation=conv,
+    )
+    response = agent([{"role": "user", "content": prompt}], stream=False)
+    return cast(str, response)
+
+
+def check_for_interaction(chat: List[AgentMessage]) -> bool:
+    return (
+        len(chat) > 2
+        and chat[-2].role == "interaction"
+        and chat[-1].role == "interaction_response"
+    )
 
 
 def maybe_run_action(
@@ -84,8 +89,10 @@ def maybe_run_action(
     chat: List[AgentMessage],
     code_interpreter: Optional[CodeInterpreter] = None,
 ) -> Optional[List[AgentMessage]]:
+    extracted_chat, final_code = extract_conversation(chat)
+    # only keep last 5 messages to keep context recent and not overwhelm LLM
+    extracted_chat = extracted_chat[-5:]
     if action == "generate_or_edit_vision_code":
-        extracted_chat, _ = extract_conversation_for_generate_code(chat)
         # there's an issue here because coder.generate_code will send it's code_context
         # to the outside user via it's update_callback, but we don't necessarily have
         # access to that update_callback here, so we re-create the message using
@@ -105,7 +112,6 @@ def maybe_run_action(
                 )
             ]
     elif action == "edit_code":
-        extracted_chat, final_code = extract_conversation_for_generate_code(chat)
         plan_context = PlanContext(
             plan="Edit the latest code observed in the fewest steps possible according to the user's feedback.",
             instructions=[
@@ -158,14 +164,7 @@ class VisionAgentV2(Agent):
                 that will send back intermediate conversation messages.
         """
 
-        self.agent = (
-            agent
-            if agent is not None
-            else AnthropicLMM(
-                model_name="claude-3-5-sonnet-20241022",
-                temperature=0.0,
-            )
-        )
+        self.agent = agent if agent is not None else CONFIG.create_agent()
         self.coder = (
             coder
             if coder is not None

vision_agent/configs/__init__.py

@@ -0,0 +1 @@
+from .config import Config

vision_agent/configs/anthropic_config.py

@@ -0,0 +1,150 @@
+from typing import Type
+
+from pydantic import BaseModel, Field
+
+from vision_agent.lmm import LMM, AnthropicLMM
+
+
+class Config(BaseModel):
+    # for vision_agent_v2
+    agent: Type[LMM] = Field(default=AnthropicLMM)
+    agent_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 0.0,
+            "image_size": 768,
+        }
+    )
+
+    # for vision_agent_planner_v2
+    planner: Type[LMM] = Field(default=AnthropicLMM)
+    planner_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 0.0,
+            "image_size": 768,
+        }
+    )
+
+    # for vision_agent_planner_v2
+    summarizer: Type[LMM] = Field(default=AnthropicLMM)
+    summarizer_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 0.0,
+            "image_size": 768,
+        }
+    )
+
+    # for vision_agent_planner_v2
+    critic: Type[LMM] = Field(default=AnthropicLMM)
+    critic_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 0.0,
+            "image_size": 768,
+        }
+    )
+
+    # for vision_agent_coder_v2
+    coder: Type[LMM] = Field(default=AnthropicLMM)
+    coder_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 0.0,
+            "image_size": 768,
+        }
+    )
+
+    # for vision_agent_coder_v2
+    tester: Type[LMM] = Field(default=AnthropicLMM)
+    tester_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 0.0,
+            "image_size": 768,
+        }
+    )
+
+    # for vision_agent_coder_v2
+    debugger: Type[LMM] = Field(default=AnthropicLMM)
+    debugger_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 0.0,
+            "image_size": 768,
+        }
+    )
+
+    # for get_tool_for_task
+    tool_tester: Type[LMM] = Field(default=AnthropicLMM)
+    tool_tester_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 1.0,
+            "image_size": 768,
+        }
+    )
+
+    # for get_tool_for_task
+    tool_chooser: Type[LMM] = Field(default=AnthropicLMM)
+    tool_chooser_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 1.0,
+            "image_size": 768,
+        }
+    )
+
+    # for suggestions module
+    suggester: Type[LMM] = Field(default=AnthropicLMM)
+    suggester_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 1.0,
+            "image_size": 768,
+        }
+    )
+
+    # for vqa module
+    vqa: Type[LMM] = Field(default=AnthropicLMM)
+    vqa_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 1.0,
+            "image_size": 768,
+        }
+    )
+
+    def create_agent(self) -> LMM:
+        return self.agent(**self.agent_kwargs)
+
+    def create_planner(self) -> LMM:
+        return self.planner(**self.planner_kwargs)
+
+    def create_summarizer(self) -> LMM:
+        return self.summarizer(**self.summarizer_kwargs)
+
+    def create_critic(self) -> LMM:
+        return self.critic(**self.critic_kwargs)
+
+    def create_coder(self) -> LMM:
+        return self.coder(**self.coder_kwargs)
+
+    def create_tester(self) -> LMM:
+        return self.tester(**self.tester_kwargs)
+
+    def create_debugger(self) -> LMM:
+        return self.debugger(**self.debugger_kwargs)
+
+    def create_tool_tester(self) -> LMM:
+        return self.tool_tester(**self.tool_tester_kwargs)
+
+    def create_tool_chooser(self) -> LMM:
+        return self.tool_chooser(**self.tool_chooser_kwargs)
+
+    def create_suggester(self) -> LMM:
+        return self.suggester(**self.suggester_kwargs)
+
+    def create_vqa(self) -> LMM:
+        return self.vqa(**self.vqa_kwargs)

vision_agent/configs/anthropic_openai_config.py

@@ -0,0 +1,150 @@
+from typing import Type
+
+from pydantic import BaseModel, Field
+
+from vision_agent.lmm import LMM, AnthropicLMM, OpenAILMM
+
+
+class Config(BaseModel):
+    # for vision_agent_v2
+    agent: Type[LMM] = Field(default=AnthropicLMM)
+    agent_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 0.0,
+            "image_size": 768,
+        }
+    )
+
+    # for vision_agent_planner_v2
+    planner: Type[LMM] = Field(default=AnthropicLMM)
+    planner_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 0.0,
+            "image_size": 768,
+        }
+    )
+
+    # for vision_agent_planner_v2
+    summarizer: Type[LMM] = Field(default=OpenAILMM)
+    summarizer_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "o1",
+            "temperature": 1.0,  # o1 has fixed temperature
+            "image_size": 768,
+        }
+    )
+
+    # for vision_agent_planner_v2
+    critic: Type[LMM] = Field(default=AnthropicLMM)
+    critic_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 0.0,
+            "image_size": 768,
+        }
+    )
+
+    # for vision_agent_coder_v2
+    coder: Type[LMM] = Field(default=AnthropicLMM)
+    coder_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 0.0,
+            "image_size": 768,
+        }
+    )
+
+    # for vision_agent_coder_v2
+    tester: Type[LMM] = Field(default=AnthropicLMM)
+    tester_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 0.0,
+            "image_size": 768,
+        }
+    )
+
+    # for vision_agent_coder_v2
+    debugger: Type[LMM] = Field(default=AnthropicLMM)
+    debugger_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 0.0,
+            "image_size": 768,
+        }
+    )
+
+    # for get_tool_for_task
+    tool_tester: Type[LMM] = Field(default=AnthropicLMM)
+    tool_tester_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 0.0,
+            "image_size": 768,
+        }
+    )
+
+    # for get_tool_for_task
+    tool_chooser: Type[LMM] = Field(default=AnthropicLMM)
+    tool_chooser_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 1.0,
+            "image_size": 768,
+        }
+    )
+
+    # for suggestions module
+    suggester: Type[LMM] = Field(default=AnthropicLMM)
+    suggester_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 1.0,
+            "image_size": 768,
+        }
+    )
+
+    # for vqa module
+    vqa: Type[LMM] = Field(default=AnthropicLMM)
+    vqa_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 0.0,
+            "image_size": 768,
+        }
+    )
+
+    def create_agent(self) -> LMM:
+        return self.agent(**self.agent_kwargs)
+
+    def create_planner(self) -> LMM:
+        return self.planner(**self.planner_kwargs)
+
+    def create_summarizer(self) -> LMM:
+        return self.summarizer(**self.summarizer_kwargs)
+
+    def create_critic(self) -> LMM:
+        return self.critic(**self.critic_kwargs)
+
+    def create_coder(self) -> LMM:
+        return self.coder(**self.coder_kwargs)
+
+    def create_tester(self) -> LMM:
+        return self.tester(**self.tester_kwargs)
+
+    def create_debugger(self) -> LMM:
+        return self.debugger(**self.debugger_kwargs)
+
+    def create_tool_tester(self) -> LMM:
+        return self.tool_tester(**self.tool_tester_kwargs)
+
+    def create_tool_chooser(self) -> LMM:
+        return self.tool_chooser(**self.tool_chooser_kwargs)
+
+    def create_suggester(self) -> LMM:
+        return self.suggester(**self.suggester_kwargs)
+
+    def create_vqa(self) -> LMM:
+        return self.vqa(**self.vqa_kwargs)

vision_agent/configs/config.py

@@ -0,0 +1,150 @@
+from typing import Type
+
+from pydantic import BaseModel, Field
+
+from vision_agent.lmm import LMM, AnthropicLMM, OpenAILMM
+
+
+class Config(BaseModel):
+    # for vision_agent_v2
+    agent: Type[LMM] = Field(default=AnthropicLMM)
+    agent_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 0.0,
+            "image_size": 768,
+        }
+    )
+
+    # for vision_agent_planner_v2
+    planner: Type[LMM] = Field(default=AnthropicLMM)
+    planner_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 0.0,
+            "image_size": 768,
+        }
+    )
+
+    # for vision_agent_planner_v2
+    summarizer: Type[LMM] = Field(default=OpenAILMM)
+    summarizer_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "o1",
+            "temperature": 1.0,  # o1 has fixed temperature
+            "image_size": 768,
+        }
+    )
+
+    # for vision_agent_planner_v2
+    critic: Type[LMM] = Field(default=AnthropicLMM)
+    critic_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 0.0,
+            "image_size": 768,
+        }
+    )
+
+    # for vision_agent_coder_v2
+    coder: Type[LMM] = Field(default=AnthropicLMM)
+    coder_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 0.0,
+            "image_size": 768,
+        }
+    )
+
+    # for vision_agent_coder_v2
+    tester: Type[LMM] = Field(default=AnthropicLMM)
+    tester_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 0.0,
+            "image_size": 768,
+        }
+    )
+
+    # for vision_agent_coder_v2
+    debugger: Type[LMM] = Field(default=AnthropicLMM)
+    debugger_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 0.0,
+            "image_size": 768,
+        }
+    )
+
+    # for get_tool_for_task
+    tool_tester: Type[LMM] = Field(default=AnthropicLMM)
+    tool_tester_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 0.0,
+            "image_size": 768,
+        }
+    )
+
+    # for get_tool_for_task
+    tool_chooser: Type[LMM] = Field(default=AnthropicLMM)
+    tool_chooser_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 1.0,
+            "image_size": 768,
+        }
+    )
+
+    # for suggestions module
+    suggester: Type[LMM] = Field(default=AnthropicLMM)
+    suggester_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 1.0,
+            "image_size": 768,
+        }
+    )
+
+    # for vqa module
+    vqa: Type[LMM] = Field(default=AnthropicLMM)
+    vqa_kwargs: dict = Field(
+        default_factory=lambda: {
+            "model_name": "claude-3-5-sonnet-20241022",
+            "temperature": 0.0,
+            "image_size": 768,
+        }
+    )
+
+    def create_agent(self) -> LMM:
+        return self.agent(**self.agent_kwargs)
+
+    def create_planner(self) -> LMM:
+        return self.planner(**self.planner_kwargs)
+
+    def create_summarizer(self) -> LMM:
+        return self.summarizer(**self.summarizer_kwargs)
+
+    def create_critic(self) -> LMM:
+        return self.critic(**self.critic_kwargs)
+
+    def create_coder(self) -> LMM:
+        return self.coder(**self.coder_kwargs)
+
+    def create_tester(self) -> LMM:
+        return self.tester(**self.tester_kwargs)
+
+    def create_debugger(self) -> LMM:
+        return self.debugger(**self.debugger_kwargs)
+
+    def create_tool_tester(self) -> LMM:
+        return self.tool_tester(**self.tool_tester_kwargs)
+
+    def create_tool_chooser(self) -> LMM:
+        return self.tool_chooser(**self.tool_chooser_kwargs)
+
+    def create_suggester(self) -> LMM:
+        return self.suggester(**self.suggester_kwargs)
+
+    def create_vqa(self) -> LMM:
+        return self.vqa(**self.vqa_kwargs)