oagi-core 0.10.3__py3-none-any.whl → 0.12.0__py3-none-any.whl

oagi/agent/tasker/planner.py

@@ -11,7 +11,7 @@ from typing import Any
 
 from ...client import AsyncClient
 from ...constants import DEFAULT_REFLECTION_INTERVAL
-from ...types import URL, Image
+from ...types import URL, Image, extract_uuid_from_url
 from .memory import PlannerMemory
 from .models import Action, PlannerOutput, ReflectionOutput
 
@@ -122,7 +122,7 @@ class Planner:
         screenshot: Image | URL | None = None,
         memory: PlannerMemory | None = None,
         todo_index: int | None = None,
-    ) -> PlannerOutput:
+    ) -> tuple[PlannerOutput, str | None]:
         """Generate initial plan for a todo.
 
         Args:
@@ -133,16 +133,21 @@ class Planner:
             todo_index: Optional todo index for formatting internal context
 
         Returns:
-            PlannerOutput with instruction, reasoning, and optional subtodos
+            Tuple of (PlannerOutput, request_id) where request_id is from API response
         """
         # Ensure we have a client
         client = self._ensure_client()
 
-        # Upload screenshot if provided
+        # Get screenshot UUID - either extract from URL or upload
         screenshot_uuid = None
         if screenshot:
-            upload_response = await client.put_s3_presigned_url(screenshot)
-            screenshot_uuid = upload_response.uuid
+            # Check if screenshot is already a URL (already uploaded to S3)
+            if isinstance(screenshot, str):
+                screenshot_uuid = extract_uuid_from_url(screenshot)
+            # If not a URL or UUID extraction failed, upload the image
+            if not screenshot_uuid:
+                upload_response = await client.put_s3_presigned_url(screenshot)
+                screenshot_uuid = upload_response.uuid
 
         # Extract memory data if provided
         (
@@ -165,8 +170,8 @@ class Planner:
             current_screenshot=screenshot_uuid,
         )
 
-        # Parse response
-        return self._parse_planner_output(response.response)
+        # Parse response and return with request_id
+        return self._parse_planner_output(response.response), response.request_id
 
     async def reflect(
         self,
@@ -177,7 +182,7 @@ class Planner:
         todo_index: int | None = None,
         current_instruction: str | None = None,
         reflection_interval: int = DEFAULT_REFLECTION_INTERVAL,
-    ) -> ReflectionOutput:
+    ) -> tuple[ReflectionOutput, str | None]:
         """Reflect on recent actions and progress.
 
         Args:
@@ -190,16 +195,21 @@ class Planner:
             reflection_interval: Window size for recent actions/screenshots
 
         Returns:
-            ReflectionOutput with continuation decision and reasoning
+            Tuple of (ReflectionOutput, request_id) where request_id is from API response
         """
         # Ensure we have a client
         client = self._ensure_client()
 
-        # Upload screenshot if provided
+        # Get screenshot UUID - either extract from URL or upload
         result_screenshot_uuid = None
         if screenshot:
-            upload_response = await client.put_s3_presigned_url(screenshot)
-            result_screenshot_uuid = upload_response.uuid
+            # Check if screenshot is already a URL (already uploaded to S3)
+            if isinstance(screenshot, str):
+                result_screenshot_uuid = extract_uuid_from_url(screenshot)
+            # If not a URL or UUID extraction failed, upload the image
+            if not result_screenshot_uuid:
+                upload_response = await client.put_s3_presigned_url(screenshot)
+                result_screenshot_uuid = upload_response.uuid
 
         # Extract memory data if provided
         (
@@ -250,8 +260,8 @@ class Planner:
             prior_notes=prior_notes,
         )
 
-        # Parse response
-        return self._parse_reflection_output(response.response)
+        # Parse response and return with request_id
+        return self._parse_reflection_output(response.response), response.request_id
 
     async def summarize(
         self,
@@ -259,7 +269,7 @@ class Planner:
         context: dict[str, Any],
         memory: PlannerMemory | None = None,
         todo_index: int | None = None,
-    ) -> str:
+    ) -> tuple[str, str | None]:
         """Generate execution summary.
 
         Args:
@@ -269,7 +279,7 @@ class Planner:
             todo_index: Optional todo index for formatting internal context
 
         Returns:
-            String summary of the execution
+            Tuple of (summary string, request_id) where request_id is from API response
         """
         # Ensure we have a client
         client = self._ensure_client()
@@ -304,9 +314,11 @@ class Planner:
         # Parse response and extract summary
         try:
             result = json.loads(response.response)
-            return result.get("task_summary", response.response)
+            summary = result.get("task_summary", response.response)
         except json.JSONDecodeError:
-            return response.response
+            summary = response.response
+
+        return summary, response.request_id
 
     def _format_execution_notes(self, context: dict[str, Any]) -> str:
         """Format execution history notes.

oagi/agent/tasker/taskee_agent.py

@@ -19,6 +19,7 @@ from oagi.constants import (
     DEFAULT_TEMPERATURE,
     MODEL_ACTOR,
 )
+from oagi.handler import reset_handler
 from oagi.types import (
     URL,
     ActionEvent,
@@ -28,6 +29,7 @@ from oagi.types import (
     Image,
     PlanEvent,
     StepEvent,
+    extract_uuid_from_url,
 )
 
 from ..protocol import AsyncAgent
@@ -121,6 +123,9 @@ class TaskeeAgent(AsyncAgent):
         Returns:
             True if successful, False otherwise
         """
+        # Reset handler state at todo execution start
+        reset_handler(action_handler)
+
         self.current_todo = instruction
         self.actions = []
         self.total_actions = 0
@@ -195,7 +200,7 @@ class TaskeeAgent(AsyncAgent):
         context = self._get_context()
 
         # Generate plan using LLM planner
-        plan_output = await self.planner.initial_plan(
+        plan_output, request_id = await self.planner.initial_plan(
             self.current_todo,
             context,
             screenshot,
@@ -219,6 +224,7 @@ class TaskeeAgent(AsyncAgent):
                     image=_serialize_image(screenshot),
                     reasoning=plan_output.reasoning,
                     result=plan_output.instruction,
+                    request_id=request_id,
                 )
             )
 
@@ -256,11 +262,21 @@ class TaskeeAgent(AsyncAgent):
             # Capture screenshot
             screenshot = await image_provider()
 
-            # Upload screenshot first to get UUID (avoids re-upload in actor.step)
+            # Get screenshot UUID - either extract from URL or upload
             try:
-                upload_response = await client.put_s3_presigned_url(screenshot)
-                screenshot_uuid = upload_response.uuid
-                screenshot_url = upload_response.download_url
+                screenshot_uuid = None
+                screenshot_url = None
+
+                # Check if screenshot is already a URL (from SocketIOImageProvider)
+                if isinstance(screenshot, str):
+                    screenshot_uuid = extract_uuid_from_url(screenshot)
+                    screenshot_url = screenshot
+
+                # If not a URL or UUID extraction failed, upload the image
+                if not screenshot_uuid:
+                    upload_response = await client.put_s3_presigned_url(screenshot)
+                    screenshot_uuid = upload_response.uuid
+                    screenshot_url = upload_response.download_url
             except Exception as e:
                 logger.error(f"Error uploading screenshot: {e}")
                 self._record_action(
@@ -294,6 +310,7 @@ class TaskeeAgent(AsyncAgent):
                         step_num=self.total_actions + 1,
                         image=_serialize_image(screenshot),
                         step=step,
+                        task_id=self.actor.task_id,
                     )
                 )
 
@@ -378,7 +395,7 @@ class TaskeeAgent(AsyncAgent):
         recent_actions = self.actions[-self.since_reflection :]
 
         # Reflect using planner
-        reflection = await self.planner.reflect(
+        reflection, request_id = await self.planner.reflect(
             recent_actions,
             context,
             screenshot,
@@ -409,6 +426,7 @@ class TaskeeAgent(AsyncAgent):
                     image=_serialize_image(screenshot),
                     reasoning=reflection.reasoning,
                     result=decision,
+                    request_id=request_id,
                 )
             )
 
@@ -441,7 +459,7 @@ class TaskeeAgent(AsyncAgent):
         context = self._get_context()
         context["current_todo"] = self.current_todo
 
-        summary = await self.planner.summarize(
+        summary, request_id = await self.planner.summarize(
             self.actions,
             context,
             memory=self.external_memory,
@@ -463,6 +481,7 @@ class TaskeeAgent(AsyncAgent):
                     image=None,
                     reasoning=summary,
                     result=None,
+                    request_id=request_id,
                 )
             )
 

oagi/agent/tasker/tasker_agent.py

@@ -16,6 +16,7 @@ from oagi.constants import (
     DEFAULT_TEMPERATURE,
     MODEL_ACTOR,
 )
+from oagi.handler import reset_handler
 from oagi.types import AsyncActionHandler, AsyncImageProvider, AsyncObserver, SplitEvent
 
 from ..protocol import AsyncAgent
@@ -112,6 +113,9 @@ class TaskerAgent(AsyncAgent):
         Returns:
             True if all todos completed successfully, False otherwise
         """
+        # Reset handler state at automation start
+        reset_handler(action_handler)
+
         overall_success = True
 
         # Execute todos until none remain

oagi/cli/agent.py

@@ -17,12 +17,9 @@ from oagi.agent.observer import AsyncAgentObserver
 from oagi.constants import (
     API_KEY_HELP_URL,
     DEFAULT_BASE_URL,
-    DEFAULT_MAX_STEPS,
     DEFAULT_MAX_STEPS_THINKER,
     DEFAULT_STEP_DELAY,
-    DEFAULT_TEMPERATURE,
     MODE_ACTOR,
-    MODEL_ACTOR,
     MODEL_THINKER,
 )
 from oagi.exceptions import check_optional_dependency
@@ -40,22 +37,30 @@ def add_agent_parser(subparsers: argparse._SubParsersAction) -> None:
         "run", help="Run an agent with the given instruction"
     )
     run_parser.add_argument(
-        "instruction", type=str, help="Task instruction for the agent to execute"
+        "instruction",
+        type=str,
+        nargs="?",
+        default="",
+        help="Task instruction for the agent to execute (optional for pre-configured modes)",
     )
     run_parser.add_argument(
-        "--model", type=str, help=f"Model to use (default: {MODEL_ACTOR})"
+        "--model", type=str, help="Model to use (default: determined by mode)"
     )
     run_parser.add_argument(
-        "--max-steps", type=int, help="Maximum number of steps (default: 20)"
+        "--max-steps",
+        type=int,
+        help="Maximum number of steps (default: determined by mode)",
     )
     run_parser.add_argument(
-        "--temperature", type=float, help="Sampling temperature (default: 0.5)"
+        "--temperature",
+        type=float,
+        help="Sampling temperature (default: determined by mode)",
     )
     run_parser.add_argument(
         "--mode",
         type=str,
         default=MODE_ACTOR,
-        help=f"Agent mode to use (default: {MODE_ACTOR}). Available modes: actor, planner",
+        help=f"Agent mode to use (default: {MODE_ACTOR}). Use 'oagi agent modes' to list available modes",
     )
     run_parser.add_argument(
         "--oagi-api-key", type=str, help="OAGI API key (default: OAGI_API_KEY env var)"
@@ -82,6 +87,9 @@ def add_agent_parser(subparsers: argparse._SubParsersAction) -> None:
         help=f"Delay in seconds after each step before next screenshot (default: {DEFAULT_STEP_DELAY})",
     )
 
+    # agent modes command
+    agent_subparsers.add_parser("modes", help="List available agent modes")
+
     # agent permission command
     agent_subparsers.add_parser(
         "permission",
@@ -92,10 +100,22 @@ def add_agent_parser(subparsers: argparse._SubParsersAction) -> None:
 def handle_agent_command(args: argparse.Namespace) -> None:
     if args.agent_command == "run":
         run_agent(args)
+    elif args.agent_command == "modes":
+        list_modes()
     elif args.agent_command == "permission":
         check_permissions()
 
 
+def list_modes() -> None:
+    """List all available agent modes."""
+    from oagi.agent import list_agent_modes  # noqa: PLC0415
+
+    modes = list_agent_modes()
+    print("Available agent modes:")
+    for mode in modes:
+        print(f"  - {mode}")
+
+
 def check_permissions() -> None:
     """Check and request macOS permissions for screen recording and accessibility.
 
@@ -207,14 +227,6 @@ def run_agent(args: argparse.Namespace) -> None:
         sys.exit(1)
 
     base_url = args.oagi_base_url or os.getenv("OAGI_BASE_URL", DEFAULT_BASE_URL)
-    model = args.model or MODEL_ACTOR
-    default_max_steps = (
-        DEFAULT_MAX_STEPS_THINKER if model == MODEL_THINKER else DEFAULT_MAX_STEPS
-    )
-    max_steps = args.max_steps or default_max_steps
-    temperature = (
-        args.temperature if args.temperature is not None else DEFAULT_TEMPERATURE
-    )
     mode = args.mode or MODE_ACTOR
     step_delay = args.step_delay if args.step_delay is not None else DEFAULT_STEP_DELAY
     export_format = args.export
@@ -233,26 +245,38 @@ def run_agent(args: argparse.Namespace) -> None:
 
     observer = CombinedObserver()
 
-    # Create agent with observer
-    agent = create_agent(
-        mode=mode,
-        api_key=api_key,
-        base_url=base_url,
-        model=model,
-        max_steps=max_steps,
-        temperature=temperature,
-        step_observer=observer,
-        step_delay=step_delay,
-    )
+    # Build agent kwargs - only pass explicitly provided values, let factory use defaults
+    agent_kwargs = {
+        "mode": mode,
+        "api_key": api_key,
+        "base_url": base_url,
+        "step_observer": observer,
+        "step_delay": step_delay,
+    }
+    if args.model:
+        agent_kwargs["model"] = args.model
+        # If thinker model specified without max_steps, use thinker's default
+        if args.model == MODEL_THINKER and not args.max_steps:
+            agent_kwargs["max_steps"] = DEFAULT_MAX_STEPS_THINKER
+    if args.max_steps:
+        agent_kwargs["max_steps"] = args.max_steps
+    if args.temperature is not None:
+        agent_kwargs["temperature"] = args.temperature
+
+    # Create agent
+    agent = create_agent(**agent_kwargs)
 
     # Create handlers
     action_handler = AsyncPyautoguiActionHandler()
     image_provider = AsyncScreenshotMaker()
 
-    print(f"Starting agent with instruction: {args.instruction}")
+    if args.instruction:
+        print(f"Starting agent with instruction: {args.instruction}")
+    else:
+        print(f"Starting agent with mode: {mode} (using pre-configured instruction)")
     print(
-        f"Mode: {mode}, Model: {model}, Max steps: {max_steps}, "
-        f"Temperature: {temperature}, Step delay: {step_delay}s"
+        f"Mode: {mode}, Model: {agent.model}, Max steps: {agent.max_steps}, "
+        f"Temperature: {agent.temperature}, Step delay: {step_delay}s"
     )
     print("-" * 60)
 

oagi/client/async_.py

@@ -9,17 +9,19 @@
 from functools import wraps
 
 import httpx
+from httpx import AsyncHTTPTransport
+from openai import AsyncOpenAI
 
 from ..constants import (
-    API_HEALTH_ENDPOINT,
     API_V1_FILE_UPLOAD_ENDPOINT,
     API_V1_GENERATE_ENDPOINT,
-    API_V2_MESSAGE_ENDPOINT,
+    DEFAULT_MAX_RETRIES,
     HTTP_CLIENT_TIMEOUT,
 )
 from ..logging import get_logger
 from ..types import Image
-from ..types.models import GenerateResponse, LLMResponse, UploadFileResponse
+from ..types.models import GenerateResponse, UploadFileResponse, Usage
+from ..types.models.step import Step
 from .base import BaseClient
 
 logger = get_logger("async_client")
@@ -35,8 +37,7 @@ def async_log_trace_on_failure(func):
         except Exception as e:
             # Try to get response from the exception if it has one
             if (response := getattr(e, "response", None)) is not None:
-                logger.error(f"Request Id: {response.headers.get('x-request-id', '')}")
-                logger.error(f"Trace Id: {response.headers.get('x-trace-id', '')}")
+                BaseClient._log_trace_id(response)
             raise
 
     return wrapper
@@ -45,115 +46,72 @@ def async_log_trace_on_failure(func):
 class AsyncClient(BaseClient[httpx.AsyncClient]):
     """Asynchronous HTTP client for the OAGI API."""
 
-    def __init__(self, base_url: str | None = None, api_key: str | None = None):
-        super().__init__(base_url, api_key)
-        self.client = httpx.AsyncClient(base_url=self.base_url)
-        self.upload_client = httpx.AsyncClient(timeout=HTTP_CLIENT_TIMEOUT)
+    def __init__(
+        self,
+        base_url: str | None = None,
+        api_key: str | None = None,
+        max_retries: int = DEFAULT_MAX_RETRIES,
+    ):
+        super().__init__(base_url, api_key, max_retries)
+
+        # OpenAI client for chat completions (with retries)
+        self.openai_client = AsyncOpenAI(
+            api_key=self.api_key,
+            base_url=f"{self.base_url}/v1",
+            max_retries=self.max_retries,
+        )
+
+        # httpx clients for S3 uploads and other endpoints (with retries)
+        transport = AsyncHTTPTransport(retries=self.max_retries)
+        self.http_client = httpx.AsyncClient(
+            transport=transport, base_url=self.base_url
+        )
+        self.upload_client = httpx.AsyncClient(
+            transport=transport, timeout=HTTP_CLIENT_TIMEOUT
+        )
+
         logger.info(f"AsyncClient initialized with base_url: {self.base_url}")
 
     async def __aenter__(self):
         return self
 
     async def __aexit__(self, exc_type, exc_val, exc_tb):
-        await self.client.aclose()
-        await self.upload_client.aclose()
+        await self.close()
 
     async def close(self):
-        """Close the underlying httpx async clients."""
-        await self.client.aclose()
+        """Close the underlying async clients."""
+        await self.openai_client.close()
+        await self.http_client.aclose()
         await self.upload_client.aclose()
 
-    @async_log_trace_on_failure
-    async def create_message(
+    async def chat_completion(
         self,
         model: str,
-        screenshot: bytes | None = None,
-        screenshot_url: str | None = None,
-        task_description: str | None = None,
-        task_id: str | None = None,
-        instruction: str | None = None,
-        messages_history: list | None = None,
+        messages: list,
         temperature: float | None = None,
-        api_version: str | None = None,
-    ) -> "LLMResponse":
+        task_id: str | None = None,
+    ) -> tuple[Step, str, Usage | None]:
         """
-        Call the /v2/message endpoint to analyze task and screenshot
+        Call OpenAI-compatible /v1/chat/completions endpoint.
 
         Args:
-            model: The model to use for task analysis
-            screenshot: Screenshot image bytes (mutually exclusive with screenshot_url)
-            screenshot_url: Direct URL to screenshot (mutually exclusive with screenshot)
-            task_description: Description of the task (required for new sessions)
-            task_id: Task ID for continuing existing task
-            instruction: Additional instruction when continuing a session
-            messages_history: OpenAI-compatible chat message history
-            temperature: Sampling temperature (0.0-2.0) for LLM inference
-            api_version: API version header
+            model: Model to use for inference
+            messages: Full message history (OpenAI-compatible format)
+            temperature: Sampling temperature (0.0-2.0)
+            task_id: Optional task ID for multi-turn conversations
 
         Returns:
-            LLMResponse: The response from the API
-
-        Raises:
-            ValueError: If both or neither screenshot and screenshot_url are provided
-            httpx.HTTPStatusError: For HTTP error responses
+            Tuple of (Step, raw_output, Usage)
+            - Step: Parsed actions and reasoning
+            - raw_output: Raw model output string (for message history)
+            - Usage: Token usage statistics (or None if not available)
         """
-        # Validate that exactly one is provided
-        if (screenshot is None) == (screenshot_url is None):
-            raise ValueError(
-                "Exactly one of 'screenshot' or 'screenshot_url' must be provided"
-            )
-
-        self._log_request_info(model, task_description, task_id)
-
-        # Upload screenshot to S3 if bytes provided, otherwise use URL directly
-        upload_file_response = None
-        if screenshot is not None:
-            upload_file_response = await self.put_s3_presigned_url(
-                screenshot, api_version
-            )
-
-        # Prepare message payload
-        headers, payload = self._prepare_message_payload(
-            model=model,
-            upload_file_response=upload_file_response,
-            task_description=task_description,
-            task_id=task_id,
-            instruction=instruction,
-            messages_history=messages_history,
-            temperature=temperature,
-            api_version=api_version,
-            screenshot_url=screenshot_url,
+        logger.info(f"Making async chat completion request with model: {model}")
+        kwargs = self._build_chat_completion_kwargs(
+            model, messages, temperature, task_id
         )
-
-        # Make request
-        try:
-            response = await self.client.post(
-                API_V2_MESSAGE_ENDPOINT,
-                json=payload,
-                headers=headers,
-                timeout=self.timeout,
-            )
-            return self._process_response(response)
-        except (httpx.TimeoutException, httpx.NetworkError) as e:
-            self._handle_upload_http_errors(e)
-
-    async def health_check(self) -> dict:
-        """
-        Call the /health endpoint for health check
-
-        Returns:
-            dict: Health check response
-        """
-        logger.debug("Making async health check request")
-        try:
-            response = await self.client.get(API_HEALTH_ENDPOINT)
-            response.raise_for_status()
-            result = response.json()
-            logger.debug("Async health check successful")
-            return result
-        except httpx.HTTPStatusError as e:
-            logger.warning(f"Async health check failed: {e}")
-            raise
+        response = await self.openai_client.chat.completions.create(**kwargs)
+        return self._parse_chat_completion_response(response)
 
     async def get_s3_presigned_url(
         self,
@@ -172,7 +130,7 @@ class AsyncClient(BaseClient[httpx.AsyncClient]):
 
         try:
             headers = self._build_headers(api_version)
-            response = await self.client.get(
+            response = await self.http_client.get(
                 API_V1_FILE_UPLOAD_ENDPOINT, headers=headers, timeout=self.timeout
             )
             return self._process_upload_response(response)
@@ -292,7 +250,7 @@ class AsyncClient(BaseClient[httpx.AsyncClient]):
 
         # Make request
         try:
-            response = await self.client.post(
+            response = await self.http_client.post(
                 API_V1_GENERATE_ENDPOINT,
                 json=payload,
                 headers=headers,