code-puppy 0.0.356__py3-none-any.whl → 0.0.357__py3-none-any.whl

code_puppy/tools/browser/vqa_agent.py

@@ -2,10 +2,12 @@
 
 from __future__ import annotations
 
-from functools import lru_cache
+from collections.abc import AsyncIterable
+from typing import Any
 
 from pydantic import BaseModel, Field
-from pydantic_ai import Agent, BinaryContent
+from pydantic_ai import Agent, BinaryContent, PartDeltaEvent, PartStartEvent, RunContext
+from pydantic_ai.messages import TextPart, TextPartDelta
 
 from code_puppy.config import get_use_dbos, get_vqa_model_name
 
@@ -18,73 +20,175 @@ class VisualAnalysisResult(BaseModel):
     observations: str
 
 
-def _get_vqa_instructions() -> str:
-    """Get the system instructions for the VQA agent."""
-    return (
-        "You are a visual analysis specialist. Answer the user's question about the provided image. "
-        "Always respond using the structured schema: answer, confidence (0-1 float), observations. "
-        "Confidence reflects how certain you are about the answer. Observations should include useful, concise context."
-    )
+DEFAULT_VQA_INSTRUCTIONS = (
+    "You are a visual analysis specialist. Answer the user's question about the provided image. "
+    "Always respond using the structured schema: answer, confidence (0-1 float), observations. "
+    "Confidence reflects how certain you are about the answer. Observations should include useful, concise context."
+)
 
 
-@lru_cache(maxsize=1)
-def _load_vqa_agent(model_name: str) -> Agent[None, VisualAnalysisResult]:
-    """Create a cached agent instance for visual analysis."""
+async def run_vqa_analysis(
+    question: str,
+    image_bytes: bytes,
+    media_type: str = "image/png",
+) -> str:
+    """Execute the VQA agent asynchronously against screenshot bytes.
+
+    Follows the same pattern as agent_tools.py for prompt preparation
+    and model configuration.
+
+    Args:
+        question: The question to ask about the image.
+        image_bytes: The raw image bytes.
+        media_type: The MIME type of the image (default: "image/png").
+        system_prompt: Optional custom system prompt. If None, uses default VQA instructions.
+
+    Returns:
+        str: The answer from the VQA analysis.
+    """
+    from code_puppy import callbacks
     from code_puppy.model_factory import ModelFactory
     from code_puppy.model_utils import prepare_prompt_for_model
 
+    # Get model configuration
+    model_name = get_vqa_model_name()
     models_config = ModelFactory.load_config()
     model = ModelFactory.get_model(model_name, models_config)
 
-    # Handle claude-code models: swap instructions (prompt prepending happens in run_vqa_analysis)
-    instructions = _get_vqa_instructions()
+    # Build instructions: custom system_prompt or default VQA instructions
+    instructions = DEFAULT_VQA_INSTRUCTIONS
+
+    # Apply prompt additions (like file permission handling) - same as agent_tools.py
+    prompt_additions = callbacks.on_load_prompt()
+    if prompt_additions:
+        instructions += "\n" + "\n".join(prompt_additions)
+
+    # Handle claude-code models: swap instructions, prepend system prompt to user question
+    # Following the exact pattern from agent_tools.py
     prepared = prepare_prompt_for_model(
-        model_name, instructions, "", prepend_system_to_user=False
+        model_name, instructions, question, prepend_system_to_user=True
     )
     instructions = prepared.instructions
+    question = prepared.user_prompt
 
+    # Create the VQA agent with string output
     vqa_agent = Agent(
         model=model,
         instructions=instructions,
-        output_type=VisualAnalysisResult,
-        retries=2,
     )
 
+    # Wrap with DBOS if enabled
     if get_use_dbos():
         from pydantic_ai.durable_exec.dbos import DBOSAgent
 
-        dbos_agent = DBOSAgent(vqa_agent, name="vqa-agent")
-        return dbos_agent
+        vqa_agent = DBOSAgent(vqa_agent, name="vqa-agent")
 
-    return vqa_agent
+    # Run the agent with the image
+    result = await vqa_agent.run(
+        [
+            question,
+            BinaryContent(data=image_bytes, media_type=media_type),
+        ]
+    )
+    return result.output
 
 
-def _get_vqa_agent() -> Agent[None, VisualAnalysisResult]:
-    """Return a cached VQA agent configured with the current model."""
-    model_name = get_vqa_model_name()
-    # lru_cache keyed by model_name ensures refresh when configuration changes
-    return _load_vqa_agent(model_name)
+def _create_vqa_stream_handler(
+    accumulator: list[str],
+):
+    """Create an event stream handler that accumulates text.
+
+    Args:
+        accumulator: List to accumulate text chunks into (pass empty list).
+
+    Returns:
+        Async event stream handler function.
+    """
+
+    async def vqa_event_stream_handler(
+        ctx: RunContext,
+        events: AsyncIterable[Any],
+    ) -> None:
+        """Handle streaming events - print text as it arrives."""
+        async for event in events:
+            # Handle text part start - might have initial content
+            if isinstance(event, PartStartEvent):
+                if isinstance(event.part, TextPart) and event.part.content:
+                    accumulator.append(event.part.content)
 
+            # Handle text deltas - the streaming bits
+            elif isinstance(event, PartDeltaEvent):
+                if isinstance(event.delta, TextPartDelta) and event.delta.content_delta:
+                    accumulator.append(event.delta.content_delta)
 
-def run_vqa_analysis(
+    return vqa_event_stream_handler
+
+
+async def run_vqa_analysis_stream(
     question: str,
     image_bytes: bytes,
     media_type: str = "image/png",
-) -> VisualAnalysisResult:
-    """Execute the VQA agent synchronously against screenshot bytes."""
-    from code_puppy.model_utils import prepare_prompt_for_model
+) -> str:
+    """Execute the VQA agent with streaming output.
 
-    agent = _get_vqa_agent()
+    Streams text to console as it arrives and accumulates the full response.
 
-    # Handle claude-code models: prepend system prompt to user question
+    Args:
+        question: The question to ask about the image.
+        image_bytes: The raw image bytes.
+        media_type: The MIME type of the image (default: "image/png").
+
+    Returns:
+        str: The accumulated answer from the VQA analysis.
+    """
+    from code_puppy import callbacks
+    from code_puppy.model_factory import ModelFactory
+    from code_puppy.model_utils import prepare_prompt_for_model
+
+    # Get model configuration
     model_name = get_vqa_model_name()
-    prepared = prepare_prompt_for_model(model_name, _get_vqa_instructions(), question)
+    models_config = ModelFactory.load_config()
+    model = ModelFactory.get_model(model_name, models_config)
+
+    # Build instructions
+    instructions = DEFAULT_VQA_INSTRUCTIONS
+
+    # Apply prompt additions (like file permission handling)
+    prompt_additions = callbacks.on_load_prompt()
+    if prompt_additions:
+        instructions += "\n" + "\n".join(prompt_additions)
+
+    # Handle claude-code models: swap instructions, prepend system prompt to user question
+    prepared = prepare_prompt_for_model(
+        model_name, instructions, question, prepend_system_to_user=True
+    )
+    instructions = prepared.instructions
     question = prepared.user_prompt
 
-    result = agent.run_sync(
+    # Create the VQA agent
+    vqa_agent = Agent(
+        model=model,
+        instructions=instructions,
+    )
+
+    # Wrap with DBOS if enabled
+    if get_use_dbos():
+        from pydantic_ai.durable_exec.dbos import DBOSAgent
+
+        vqa_agent = DBOSAgent(vqa_agent, name="vqa-agent-stream")
+
+    # Accumulator for streamed text (use list to allow mutation in handler)
+    accumulated_chunks: list[str] = []
+
+    # Create the stream handler
+    stream_handler = _create_vqa_stream_handler(accumulated_chunks)
+
+    # Run the agent with event_stream_handler
+    result = await vqa_agent.run(
         [
             question,
             BinaryContent(data=image_bytes, media_type=media_type),
-        ]
+        ],
+        event_stream_handler=stream_handler,
     )
     return result.output

code_puppy/tools/command_runner.py

@@ -1071,7 +1071,6 @@ async def run_shell_command(
                 error="Another command is currently awaiting confirmation",
             )
 
-
         # Get puppy name for personalized messages
         from code_puppy.config import get_puppy_name
 

{code_puppy-0.0.356.dist-info → code_puppy-0.0.357.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-puppy
-Version: 0.0.356
+Version: 0.0.357
 Summary: Code generation agent
 Project-URL: repository, https://github.com/mpfaffenberger/code_puppy
 Project-URL: HomePage, https://github.com/mpfaffenberger/code_puppy

{code_puppy-0.0.356.dist-info → code_puppy-0.0.357.dist-info}/RECORD

@@ -4,7 +4,7 @@ code_puppy/callbacks.py,sha256=Pp0VyeXJBEtk-N_RSWr5pbveelovsdLUiJ4f11dzwGw,10775
 code_puppy/chatgpt_codex_client.py,sha256=Om0ANB_kpHubhCwNzF9ENf8RvKBqs0IYzBLl_SNw0Vk,9833
 code_puppy/claude_cache_client.py,sha256=Gl6um5ZaKpcnxOvoFSM8Lwm_Vu4-VyWz8Nli8DnRLa4,22508
 code_puppy/cli_runner.py,sha256=w5CLKgQYYaT7My3Cga2StXYol-u6DBxNzzUuhhsfhsA,34952
-code_puppy/config.py,sha256=1bga5PP_usHkxRx2Vqaz_s820YhADo9K8R5pcaE9O90,54338
+code_puppy/config.py,sha256=gwOK-WDuYBzJKwyGCFALJiW0pstiA39pRDm1O1zFek4,54528
 code_puppy/error_logging.py,sha256=a80OILCUtJhexI6a9GM-r5LqIdjvSRzggfgPp2jv1X0,3297
 code_puppy/gemini_code_assist.py,sha256=KGS7sO5OLc83nDF3xxS-QiU6vxW9vcm6hmzilu79Ef8,13867
 code_puppy/http_utils.py,sha256=H3N5Qz2B1CcsGUYOycGWAqoNMr2P1NCVluKX3aRwRqI,10358
@@ -38,8 +38,9 @@ code_puppy/agents/agent_planning.py,sha256=6q3s5qCko2FcUfaLzImOFNDi0H61WBc2PNtsO
 code_puppy/agents/agent_python_programmer.py,sha256=R-7XoGIFJ58EY9LE9mWGcQQ8gSsMzi-1HD6wigJQPL8,6846
 code_puppy/agents/agent_python_reviewer.py,sha256=J8lqzoKJlohs8NWMbgUpHXNt1bXHNIkuGjzLd9Af8qE,5854
 code_puppy/agents/agent_qa_expert.py,sha256=5Ikb4U3SZQknUEfwlHZiyZXKqnffnOTQagr_wrkUkPk,10125
-code_puppy/agents/agent_qa_kitten.py,sha256=5PeFFSwCFlTUvP6h5bGntx0xv5NmRwBiw0HnMqY8nLI,9107
+code_puppy/agents/agent_qa_kitten.py,sha256=bjQdAPL_VMjSDn012mHQgnduuQkGG0JeXuC3T1KrU6g,9372
 code_puppy/agents/agent_security_auditor.py,sha256=SpiYNA0XAsIwBj7S2_EQPRslRUmF_-b89pIJyW7DYtY,12022
+code_puppy/agents/agent_terminal_qa.py,sha256=U-iyP7OBWdAmchW_oUU8k6asH2aignTMmgqqYDyf-ms,10343
 code_puppy/agents/agent_typescript_reviewer.py,sha256=vsnpp98xg6cIoFAEJrRTUM_i4wLEWGm5nJxs6fhHobM,10275
 code_puppy/agents/base_agent.py,sha256=oKlX9CEIWSvdXyQDVi9F1jauA6rjKleY_n6044Ux5DY,73840
 code_puppy/agents/event_stream_handler.py,sha256=HM62_THZpMVnqKIB6Vbo6IwmJt6Kjoc3YbyRy2FclA4,13805
@@ -54,15 +55,15 @@ code_puppy/agents/pack/shepherd.py,sha256=gSsPAj7MV_AaiSUWNNoFLW0C932ToctJl2X9yS
 code_puppy/agents/pack/terrier.py,sha256=bw0f5g0fmIMVsVZsBYwDrIUwMhldg1U7tzGk1EHhgK0,7935
 code_puppy/agents/pack/watchdog.py,sha256=xgiyrGgMdf8CNJoZ0bPqZ_oZBc6cu6It8d_jeNONiy4,10039
 code_puppy/api/__init__.py,sha256=q0VZ5PgiCjEL2xWPVsElwAffDscJ6JtbbIhYFmw3TA4,357
-code_puppy/api/app.py,sha256=4imfbatGX0h2rIUfo4vP2H4XLOvutM0D_EJMd_dQAf8,2868
+code_puppy/api/app.py,sha256=J7a0jfSJ1kqS0t3wt0AaCEGqIroVsXtVoVcbJi0Ag5k,5456
 code_puppy/api/main.py,sha256=ycWD5_aWwdOQ2GPnqsQC-BjINNx_Qie6xFyjr9gYofI,453
 code_puppy/api/pty_manager.py,sha256=P-SB9yuMlrwfgedV6VkzWDtGBMKAxnqV-RysQubhrS8,13714
 code_puppy/api/websocket.py,sha256=j_96B2k2XQ-KGsNqGZqTjUeUH8NkDGfgSs32Bj4C634,5360
 code_puppy/api/routers/__init__.py,sha256=-ifjlIm_yeFQ27vIC6OzeQFJZjnUF1e6xbFK1aNG798,430
 code_puppy/api/routers/agents.py,sha256=Lshn89lIBi49if6GoJDBZQ2_8l7JfgAUy4gvYXlfmws,939
-code_puppy/api/routers/commands.py,sha256=kl3ouYIg6DRWgPIDmDnN6RXJzyeQqUrh8cm1jR_eG1M,5821
+code_puppy/api/routers/commands.py,sha256=HRvsYmjrUL_tha7-nKFBEJhGRfUlCspKkngwLU8TQ6Y,6513
 code_puppy/api/routers/config.py,sha256=uDUFYZqki0fQd0U5EHpfTgqlZaRFfmhPyWrIHXNBX3A,2005
-code_puppy/api/routers/sessions.py,sha256=KeTKXLj0BjPMvnSU91LmNgc0coQ7QdRRn8puqdvknsc,5346
+code_puppy/api/routers/sessions.py,sha256=GqYRT7IJYPpEdTseLF3FIpbvvD86lIqwwPswL31D9Wc,6786
 code_puppy/api/templates/terminal.html,sha256=9alh6tTbLyXPDjBvkXw8nEWPXB-m_LIceGGRYpSLuyo,13125
 code_puppy/command_line/__init__.py,sha256=y7WeRemfYppk8KVbCGeAIiTuiOszIURCDjOMZv_YRmU,45
 code_puppy/command_line/add_model_menu.py,sha256=CpURhxPvUhLHLBV_uwH1ODfJ-WAcGklvlsjEf5Vfvg4,43255
@@ -183,29 +184,34 @@ code_puppy/plugins/shell_safety/command_cache.py,sha256=adYtSPNVOZfW_6dQdtEihO6E
 code_puppy/plugins/shell_safety/register_callbacks.py,sha256=W3v664RR48Fdbbbltf_NnX22_Ahw2AvAOtvXvWc7KxQ,7322
 code_puppy/prompts/antigravity_system_prompt.md,sha256=ZaTfRyY57ttROyZMmOBtqZQu1to7sdTNTv8_0fTgPNw,6807
 code_puppy/prompts/codex_system_prompt.md,sha256=hEFTCziroLqZmqNle5kG34A8kvTteOWezCiVrAEKhE0,24400
-code_puppy/tools/__init__.py,sha256=eQY-GL2ToV9IdRKlrnWlcPLyncJyU1VGZxq9yy0twNI,6137
-code_puppy/tools/agent_tools.py,sha256=z9Uj65aqIZ4etyq6mtH05DQEbA6dWSWAN-wH8RSmdXg,24476
-code_puppy/tools/command_runner.py,sha256=OLwPUNicgiHfQj6Ux4SaVNYmX425VbMjrv-DsxZ9e_M,50996
+code_puppy/tools/__init__.py,sha256=WC1DO3OeTVSibpvIIoyfdxbeeC0oigiBSUqpmdw8G4o,7615
+code_puppy/tools/agent_tools.py,sha256=XvBQ_IPa4NHLmIA2mdyPwy9GPlYGQwhtdn-w_3i239g,25517
+code_puppy/tools/command_runner.py,sha256=Sresr_ykou_c2V1sKoNxqrqCQovKF5yDiQJ8r3E9lak,50995
 code_puppy/tools/common.py,sha256=lVtF94cn6jtC5YKfitV7L3rk37Ts2gMoHLQrqDFD2E4,46411
 code_puppy/tools/display.py,sha256=-ulDyq55178f8O_TAEmnxGoy_ZdFkbHBw-W4ul851GM,2675
 code_puppy/tools/file_modifications.py,sha256=vz9n7R0AGDSdLUArZr_55yJLkyI30M8zreAppxIx02M,29380
 code_puppy/tools/file_operations.py,sha256=CqhpuBnOFOcQCIYXOujskxq2VMLWYJhibYrH0YcPSfA,35692
 code_puppy/tools/subagent_context.py,sha256=zsiKV3B3DxZ_Y5IHHhtE-SMFDg_jMrY7Hi6r5LH--IU,4781
 code_puppy/tools/tools_content.py,sha256=bsBqW-ppd1XNAS_g50B3UHDQBWEALC1UneH6-afz1zo,2365
-code_puppy/tools/browser/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-code_puppy/tools/browser/browser_control.py,sha256=MWBv_WR_A6wmm1NJCYFGQGRyYhTLplL4J0_ECZD0IhE,8388
-code_puppy/tools/browser/browser_interactions.py,sha256=AGQkAtNl2YI9aS9jM1KgYoMywVDhPOZSV6yf1b5yeyE,16517
-code_puppy/tools/browser/browser_locators.py,sha256=U7a6fi1LdCiIImeVhm0_VKn0WZ1A2DnpR6eSroXgQkg,19114
-code_puppy/tools/browser/browser_navigation.py,sha256=sX0EIACIUVjWvod_aJpqzDznPbhrBaMRYWBapRNsHr0,7367
-code_puppy/tools/browser/browser_screenshot.py,sha256=7jeG5N4-OkpRPY3JMwOrsJjutYV8wRYF6R8sjg5LawU,8132
-code_puppy/tools/browser/browser_scripts.py,sha256=sNb8eLEyzhasy5hV4B9OjM8yIVMLVEMRcQ4K77ABjRI,14564
+code_puppy/tools/browser/__init__.py,sha256=HqP5_AKL9IuaXeGLhL_Y799DBU28QZBd2x5ISKJlprc,1097
+code_puppy/tools/browser/browser_control.py,sha256=YntpjfWTIv0TDlAO5BqTV_hDbUBw-8wmMn29K3TDQo0,8430
+code_puppy/tools/browser/browser_interactions.py,sha256=ZyJmA2-ZtIATF76uGMt08cfVaYiqg7W2-cHfAzNI0F8,16775
+code_puppy/tools/browser/browser_locators.py,sha256=sxXNm-K087poeSp7Um5Gc1sZxb7HlSZOu0F0r2b0ty8,19177
+code_puppy/tools/browser/browser_navigation.py,sha256=RJdG14UXtA6wz4PNLw2Tqeu4oUDQilOyNbyTjgIFCrY,7416
+code_puppy/tools/browser/browser_screenshot.py,sha256=POlHDG7WJbjF3uBPUD7X2elAs-CKP9Dq7UQ7UZyvZGQ,5666
+code_puppy/tools/browser/browser_screenshot_vqa.py,sha256=DBdQuV7eIaJX2Qy_liwitfakIzrcVziB-zAGIngM5GE,7349
+code_puppy/tools/browser/browser_scripts.py,sha256=CYWdQMtjKTNvJNSCkB2vGo-MOzmT_gw2oFMGtkfuzuA,14779
 code_puppy/tools/browser/browser_workflows.py,sha256=nitW42vCf0ieTX1gLabozTugNQ8phtoFzZbiAhw1V90,6491
-code_puppy/tools/browser/camoufox_manager.py,sha256=RZjGOEftE5sI_tsercUyXFSZI2wpStXf-q0PdYh2G3I,8680
-code_puppy/tools/browser/vqa_agent.py,sha256=BZzpwPinMH-dsOBMvPVYJzDZ6fu3TlIARygcwIva9w4,2917
-code_puppy-0.0.356.data/data/code_puppy/models.json,sha256=FMQdE_yvP_8y0xxt3K918UkFL9cZMYAqW1SfXcQkU_k,3105
-code_puppy-0.0.356.data/data/code_puppy/models_dev_api.json,sha256=wHjkj-IM_fx1oHki6-GqtOoCrRMR0ScK0f-Iz0UEcy8,548187
-code_puppy-0.0.356.dist-info/METADATA,sha256=Yda4z2GGJVpp8rZTMmj5XNsqnQSMRaSrNV-_x1fNgKU,27614
-code_puppy-0.0.356.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-code_puppy-0.0.356.dist-info/entry_points.txt,sha256=Tp4eQC99WY3HOKd3sdvb22vZODRq0XkZVNpXOag_KdI,91
-code_puppy-0.0.356.dist-info/licenses/LICENSE,sha256=31u8x0SPgdOq3izJX41kgFazWsM43zPEF9eskzqbJMY,1075
-code_puppy-0.0.356.dist-info/RECORD,,
+code_puppy/tools/browser/camoufox_manager.py,sha256=WIr98SrGeC5jd6jX5tjhFR6A3janqV4tq9Mbznnlh44,13920
+code_puppy/tools/browser/chromium_terminal_manager.py,sha256=w1thQ_ACb6oV45L93TSqPQD0o0cTh3FqT5I9zcOOWlM,8226
+code_puppy/tools/browser/terminal_command_tools.py,sha256=9byOZku-dwvTtCl532xt7Lumed_jTn0sLvUe_X75XCQ,19068
+code_puppy/tools/browser/terminal_screenshot_tools.py,sha256=DAqzlqOoTfQZCKKeXccElrrY7s2CwSblvBV7A6A2GYw,17224
+code_puppy/tools/browser/terminal_tools.py,sha256=F5LjVH3udSCFHmqC3O1UJLoLozZFZsEdX42jOmkqkW0,17853
+code_puppy/tools/browser/vqa_agent.py,sha256=0IbS1X3l8ADZI9pGcJbKFoN0-ZuTJa8QvHZ_hGKBKRM,6339
+code_puppy-0.0.357.data/data/code_puppy/models.json,sha256=FMQdE_yvP_8y0xxt3K918UkFL9cZMYAqW1SfXcQkU_k,3105
+code_puppy-0.0.357.data/data/code_puppy/models_dev_api.json,sha256=wHjkj-IM_fx1oHki6-GqtOoCrRMR0ScK0f-Iz0UEcy8,548187
+code_puppy-0.0.357.dist-info/METADATA,sha256=czyAX-ExI8J1C2WqFxnPakQofzpUXqX5r2-Y1WIL34E,27614
+code_puppy-0.0.357.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+code_puppy-0.0.357.dist-info/entry_points.txt,sha256=Tp4eQC99WY3HOKd3sdvb22vZODRq0XkZVNpXOag_KdI,91
+code_puppy-0.0.357.dist-info/licenses/LICENSE,sha256=31u8x0SPgdOq3izJX41kgFazWsM43zPEF9eskzqbJMY,1075
+code_puppy-0.0.357.dist-info/RECORD,,