oagi-core 0.9.2__py3-none-any.whl → 0.10.0__py3-none-any.whl

oagi/agent/tasker/taskee_agent.py

@@ -11,7 +11,16 @@ from datetime import datetime
 from typing import Any
 
 from oagi import AsyncActor
-from oagi.types import AsyncActionHandler, AsyncImageProvider, AsyncStepObserver
+from oagi.types import (
+    URL,
+    ActionEvent,
+    AsyncActionHandler,
+    AsyncImageProvider,
+    AsyncObserver,
+    Image,
+    PlanEvent,
+    StepEvent,
+)
 
 from ..protocol import AsyncAgent
 from .memory import PlannerMemory
@@ -21,6 +30,13 @@ from .planner import Planner
 logger = logging.getLogger(__name__)
 
 
+def _serialize_image(image: Image | str) -> bytes | str:
+    """Convert an image to bytes or keep URL as string."""
+    if isinstance(image, str):
+        return image
+    return image.read()
+
+
 class TaskeeAgent(AsyncAgent):
     """Executes a single todo with planning and reflection capabilities.
 
@@ -36,13 +52,13 @@ class TaskeeAgent(AsyncAgent):
         api_key: str | None = None,
         base_url: str | None = None,
         model: str = "lux-actor-1",
-        max_steps_per_subtask: int = 20,
+        max_steps: int = 20,
         reflection_interval: int = 4,
         temperature: float = 0.5,
         planner: Planner | None = None,
         external_memory: PlannerMemory | None = None,
         todo_index: int | None = None,
-        step_observer: AsyncStepObserver | None = None,
+        step_observer: AsyncObserver | None = None,
     ):
         """Initialize the taskee agent.
 
@@ -50,7 +66,7 @@ class TaskeeAgent(AsyncAgent):
             api_key: OAGI API key
             base_url: OAGI API base URL
             model: Model to use for vision tasks
-            max_steps_per_subtask: Maximum steps before reinitializing task
+            max_steps: Maximum steps before reinitializing task
             reflection_interval: Number of actions before triggering reflection
             temperature: Sampling temperature
             planner: Planner for planning and reflection
@@ -61,7 +77,7 @@ class TaskeeAgent(AsyncAgent):
         self.api_key = api_key
         self.base_url = base_url
         self.model = model
-        self.max_steps_per_subtask = max_steps_per_subtask
+        self.max_steps = max_steps
         self.reflection_interval = reflection_interval
         self.temperature = temperature
         self.planner = planner or Planner(api_key=api_key, base_url=base_url)
@@ -101,17 +117,27 @@ class TaskeeAgent(AsyncAgent):
         self.success = False
 
         try:
+            self.actor = AsyncActor(
+                api_key=self.api_key,
+                base_url=self.base_url,
+                model=self.model,
+                temperature=self.temperature,
+            )
             # Initial planning
             await self._initial_plan(image_provider)
 
+            # Initialize the actor with the task
+            await self.actor.init_task(
+                self.current_instruction, max_steps=self.max_steps
+            )
+
             # Main execution loop with reinitializations
-            max_total_steps = self.max_steps_per_subtask * 3  # Allow up to 3 reinits
-            remaining_steps = max_total_steps
+            remaining_steps = self.max_steps
 
             while remaining_steps > 0 and not self.success:
                 # Execute subtask
                 steps_taken = await self._execute_subtask(
-                    min(self.max_steps_per_subtask, remaining_steps),
+                    min(self.max_steps, remaining_steps),
                     action_handler,
                     image_provider,
                 )
@@ -174,6 +200,17 @@ class TaskeeAgent(AsyncAgent):
             result=plan_output.instruction,
         )
 
+        # Emit plan event
+        if self.step_observer:
+            await self.step_observer.on_event(
+                PlanEvent(
+                    phase="initial",
+                    image=_serialize_image(screenshot),
+                    reasoning=plan_output.reasoning,
+                    result=plan_output.instruction,
+                )
+            )
+
         # Set current instruction
         self.current_instruction = plan_output.instruction
         logger.info(f"Initial instruction: {self.current_instruction}")
@@ -201,89 +238,108 @@ class TaskeeAgent(AsyncAgent):
         """
         logger.info(f"Executing subtask with max {max_steps} steps")
 
-        # Use async with for automatic resource management
-        async with AsyncActor(
-            api_key=self.api_key,
-            base_url=self.base_url,
-            model=self.model,
-            temperature=self.temperature,
-        ) as actor:
-            # Store reference for potential cleanup in execute's finally block
-            self.actor = actor
-
-            # Initialize actor with current instruction
-            await actor.init_task(self.current_instruction)
-
-            steps_taken = 0
-            for step_num in range(max_steps):
-                # Capture screenshot
-                screenshot = await image_provider()
-
-                # Get next step from OAGI
-                try:
-                    step = await actor.step(screenshot, instruction=None)
-                except Exception as e:
-                    logger.error(f"Error getting step from OAGI: {e}")
-                    self._record_action(
-                        action_type="error",
-                        target="oagi_step",
-                        reasoning=str(e),
+        steps_taken = 0
+        client = self.planner._ensure_client()
+
+        for step_num in range(max_steps):
+            # Capture screenshot
+            screenshot = await image_provider()
+
+            # Upload screenshot first to get UUID (avoids re-upload in actor.step)
+            try:
+                upload_response = await client.put_s3_presigned_url(screenshot)
+                screenshot_uuid = upload_response.uuid
+                screenshot_url = upload_response.download_url
+            except Exception as e:
+                logger.error(f"Error uploading screenshot: {e}")
+                self._record_action(
+                    action_type="error",
+                    target="screenshot_upload",
+                    reasoning=str(e),
+                )
+                break
+
+            # Get next step from OAGI using URL (avoids re-upload)
+            try:
+                step = await self.actor.step(URL(screenshot_url), instruction=None)
+            except Exception as e:
+                logger.error(f"Error getting step from OAGI: {e}")
+                self._record_action(
+                    action_type="error",
+                    target="oagi_step",
+                    reasoning=str(e),
+                    screenshot_uuid=screenshot_uuid,
+                )
+                break
+
+            # Log reasoning
+            if step.reason:
+                logger.info(f"Step {self.total_actions + 1}: {step.reason}")
+
+            # Emit step event
+            if self.step_observer:
+                await self.step_observer.on_event(
+                    StepEvent(
+                        step_num=self.total_actions + 1,
+                        image=_serialize_image(screenshot),
+                        step=step,
                     )
-                    break
+                )
 
-                # Log reasoning
-                if step.reason:
-                    logger.info(f"Step {self.total_actions + 1}: {step.reason}")
+            # Record OAGI actions
+            if step.actions:
+                # Log actions with details
+                logger.info(f"Actions ({len(step.actions)}):")
+                for action in step.actions:
+                    count_suffix = (
+                        f" x{action.count}" if action.count and action.count > 1 else ""
+                    )
+                    logger.info(
+                        f"  [{action.type.value}] {action.argument}{count_suffix}"
+                    )
 
-                # Notify observer if present
-                if self.step_observer:
-                    await self.step_observer.on_step(
-                        self.total_actions + 1, step.reason, step.actions
+                for action in step.actions:
+                    self._record_action(
+                        action_type=action.type.lower(),
+                        target=action.argument,
+                        reasoning=step.reason,
+                        screenshot_uuid=screenshot_uuid,
                     )
 
-                # Record OAGI actions
-                if step.actions:
-                    # Log actions with details
-                    logger.info(f"Actions ({len(step.actions)}):")
-                    for action in step.actions:
-                        count_suffix = (
-                            f" x{action.count}"
-                            if action.count and action.count > 1
-                            else ""
-                        )
-                        logger.info(
-                            f"  [{action.type.value}] {action.argument}{count_suffix}"
-                        )
+                # Execute actions
+                error = None
+                try:
+                    await action_handler(step.actions)
+                except Exception as e:
+                    error = str(e)
+                    raise
 
-                    for action in step.actions:
-                        self._record_action(
-                            action_type=action.type.lower(),
-                            target=action.argument,
-                            reasoning=step.reason,
+                # Emit action event
+                if self.step_observer:
+                    await self.step_observer.on_event(
+                        ActionEvent(
+                            step_num=self.total_actions + 1,
+                            actions=step.actions,
+                            error=error,
                         )
+                    )
 
-                    # Execute actions
-                    await action_handler(step.actions)
-                    self.total_actions += len(step.actions)
-                    self.since_reflection += len(step.actions)
+                self.total_actions += len(step.actions)
+                self.since_reflection += len(step.actions)
 
-                steps_taken += 1
+            steps_taken += 1
 
-                # Check if task is complete
-                if step.stop:
-                    logger.info("OAGI signaled task completion")
-                    self.success = True
-                    break
+            # Check if task is complete
+            if step.stop:
+                logger.info("OAGI signaled task completion")
+                break
 
-                # Check if reflection is needed
-                if self.since_reflection >= self.reflection_interval:
-                    logger.info("Reflection interval reached")
-                    break
+            # Check if reflection is needed
+            if self.since_reflection >= self.reflection_interval:
+                logger.info("Reflection interval reached")
+                break
 
-            # Actor will be automatically closed by async with context manager
-            # Clear reference after context manager closes it
-            self.actor = None
-            return steps_taken
+        return steps_taken
 
     async def _reflect_and_decide(self, image_provider: AsyncImageProvider) -> bool:
         """Reflect on progress and decide whether to continue.
@@ -314,6 +370,7 @@ class TaskeeAgent(AsyncAgent):
             memory=self.external_memory,
             todo_index=self.todo_index,
             current_instruction=self.current_instruction,
+            reflection_interval=self.reflection_interval,
         )
 
         # Record reflection
@@ -324,6 +381,22 @@ class TaskeeAgent(AsyncAgent):
             result=("continue" if reflection.continue_current else "pivot"),
         )
 
+        # Emit plan event for reflection
+        if self.step_observer:
+            decision = (
+                "success"
+                if reflection.success_assessment
+                else ("continue" if reflection.continue_current else "pivot")
+            )
+            await self.step_observer.on_event(
+                PlanEvent(
+                    phase="reflection",
+                    image=_serialize_image(screenshot),
+                    reasoning=reflection.reasoning,
+                    result=decision,
+                )
+            )
+
         # Update success assessment
         if reflection.success_assessment:
             self.success = True
@@ -337,6 +410,11 @@ class TaskeeAgent(AsyncAgent):
         if not reflection.continue_current and reflection.new_instruction:
             logger.info(f"Pivoting to new instruction: {reflection.new_instruction}")
             self.current_instruction = reflection.new_instruction
+
+            # the following line create a new actor
+            await self.actor.init_task(
+                self.current_instruction, max_steps=self.max_steps
+            )
             return True
 
         return reflection.continue_current
@@ -362,6 +440,17 @@ class TaskeeAgent(AsyncAgent):
             reasoning=summary,
         )
 
+        # Emit plan event for summary
+        if self.step_observer:
+            await self.step_observer.on_event(
+                PlanEvent(
+                    phase="summary",
+                    image=None,
+                    reasoning=summary,
+                    result=None,
+                )
+            )
+
         logger.info(f"Execution summary: {summary}")
 
     def _record_action(
@@ -370,6 +459,7 @@ class TaskeeAgent(AsyncAgent):
         target: str | None,
         reasoning: str | None = None,
         result: str | None = None,
+        screenshot_uuid: str | None = None,
     ) -> None:
         """Record an action to the history.
 
@@ -378,6 +468,7 @@ class TaskeeAgent(AsyncAgent):
             target: Target of the action
             reasoning: Reasoning for the action
             result: Result of the action
+            screenshot_uuid: UUID of uploaded screenshot for this action
         """
         action = Action(
             timestamp=datetime.now().isoformat(),
@@ -386,6 +477,7 @@ class TaskeeAgent(AsyncAgent):
             reasoning=reasoning,
             result=result,
             details={},
+            screenshot_uuid=screenshot_uuid,
         )
         self.actions.append(action)
 

oagi/agent/tasker/tasker_agent.py

@@ -9,7 +9,7 @@
 import logging
 from typing import Any
 
-from oagi.types import AsyncActionHandler, AsyncImageProvider, AsyncStepObserver
+from oagi.types import AsyncActionHandler, AsyncImageProvider, AsyncObserver, SplitEvent
 
 from ..protocol import AsyncAgent
 from .memory import PlannerMemory
@@ -39,7 +39,7 @@ class TaskerAgent(AsyncAgent):
         temperature: float = 0.5,
         reflection_interval: int = 4,
         planner: Planner | None = None,
-        step_observer: AsyncStepObserver | None = None,
+        step_observer: AsyncObserver | None = None,
     ):
         """Initialize the tasker agent.
 
@@ -100,18 +100,13 @@ class TaskerAgent(AsyncAgent):
         or a failure occurs.
 
         Args:
-            instruction: Overall instruction (can be same as task description)
+            instruction: Not used in TaskerAgent
             action_handler: Handler for executing actions
             image_provider: Provider for capturing screenshots
 
         Returns:
             True if all todos completed successfully, False otherwise
         """
-        # If task not set, use instruction as task description
-        if not self.memory.task_description:
-            logger.warning("Task not set, using instruction as task description")
-            self.memory.task_description = instruction
-
         overall_success = True
 
         # Execute todos until none remain
@@ -127,6 +122,14 @@ class TaskerAgent(AsyncAgent):
             todo, todo_index = todo_info
             logger.info(f"Executing todo {todo_index}: {todo.description}")
 
+            # Emit split event at the start of todo
+            if self.step_observer:
+                await self.step_observer.on_event(
+                    SplitEvent(
+                        label=f"Start of todo {todo_index + 1}: {todo.description}"
+                    )
+                )
+
             # Execute the todo
             success = await self._execute_todo(
                 todo_index,
@@ -134,6 +137,14 @@ class TaskerAgent(AsyncAgent):
                 image_provider,
             )
 
+            # Emit split event after each todo
+            if self.step_observer:
+                await self.step_observer.on_event(
+                    SplitEvent(
+                        label=f"End of todo {todo_index + 1}: {todo.description}"
+                    )
+                )
+
             if not success:
                 logger.warning(f"Todo {todo_index} failed")
                 overall_success = False
@@ -171,7 +182,7 @@ class TaskerAgent(AsyncAgent):
             api_key=self.api_key,
             base_url=self.base_url,
             model=self.model,
-            max_steps_per_subtask=20,  # Smaller steps per subtask
+            max_steps=self.max_steps,  # Smaller steps per subtask
             reflection_interval=self.reflection_interval,
             temperature=self.temperature,
             planner=self.planner,

oagi/cli/agent.py

@@ -13,6 +13,7 @@ import sys
 import time
 import traceback
 
+from oagi.agent.observer import AsyncAgentObserver
 from oagi.exceptions import check_optional_dependency
 
 from .display import display_step_table
@@ -53,6 +54,17 @@ def add_agent_parser(subparsers: argparse._SubParsersAction) -> None:
         type=str,
         help="OAGI base URL (default: https://api.agiopen.org, or OAGI_BASE_URL env var)",
     )
+    run_parser.add_argument(
+        "--export",
+        type=str,
+        choices=["markdown", "html", "json"],
+        help="Export execution history to file (markdown, html, or json)",
+    )
+    run_parser.add_argument(
+        "--export-file",
+        type=str,
+        help="Output file path for export (default: execution_report.[md|html|json])",
+    )
 
 
 def handle_agent_command(args: argparse.Namespace) -> None:
@@ -85,11 +97,23 @@ def run_agent(args: argparse.Namespace) -> None:
     max_steps = args.max_steps or 20
     temperature = args.temperature if args.temperature is not None else 0.5
     mode = args.mode or "actor"
+    export_format = args.export
+    export_file = args.export_file
 
-    # Create step tracker
+    # Create observers
     step_tracker = StepTracker()
+    agent_observer = AsyncAgentObserver() if export_format else None
+
+    # Use a combined observer that forwards to both
+    class CombinedObserver:
+        async def on_event(self, event):
+            await step_tracker.on_event(event)
+            if agent_observer:
+                await agent_observer.on_event(event)
 
-    # Create agent with step tracker
+    observer = CombinedObserver()
+
+    # Create agent with observer
     agent = create_agent(
         mode=mode,
         api_key=api_key,
@@ -97,7 +121,7 @@ def run_agent(args: argparse.Namespace) -> None:
         model=model,
         max_steps=max_steps,
         temperature=temperature,
-        step_observer=step_tracker,
+        step_observer=observer,
     )
 
     # Create handlers
@@ -137,6 +161,21 @@ def run_agent(args: argparse.Namespace) -> None:
         else:
             print("\nNo steps were executed.")
 
+        # Export if requested
+        if export_format and agent_observer:
+            # Determine output file path
+            if export_file:
+                output_path = export_file
+            else:
+                ext_map = {"markdown": "md", "html": "html", "json": "json"}
+                output_path = f"execution_report.{ext_map[export_format]}"
+
+            try:
+                agent_observer.export(export_format, output_path)
+                print(f"\nExecution history exported to: {output_path}")
+            except Exception as e:
+                print(f"\nError exporting execution history: {e}", file=sys.stderr)
+
         if interrupted:
             sys.exit(130)
         elif not success:

oagi/cli/tracking.py

@@ -9,7 +9,7 @@
 from dataclasses import dataclass
 from datetime import datetime
 
-from oagi.types import Action
+from oagi.types import Action, ActionEvent, ObserverEvent, StepEvent
 
 
 @dataclass
@@ -23,23 +23,33 @@ class StepData:
 
 
 class StepTracker:
-    """Tracks agent step execution by implementing AsyncStepObserver protocol."""
+    """Tracks agent step execution by implementing AsyncObserver protocol."""
 
     def __init__(self):
         self.steps: list[StepData] = []
 
-    async def on_step(
-        self,
-        step_num: int,
-        reasoning: str | None,
-        actions: list[Action],
-    ) -> None:
-        step_data = StepData(
-            step_num=step_num,
-            timestamp=datetime.now(),
-            reasoning=reasoning,
-            actions=actions,
-            action_count=len(actions),
-            status="running",
-        )
-        self.steps.append(step_data)
+    async def on_event(self, event: ObserverEvent) -> None:
+        """Handle observer events.
+
+        Args:
+            event: The observer event to handle.
+        """
+        match event:
+            case StepEvent():
+                step_data = StepData(
+                    step_num=event.step_num,
+                    timestamp=event.timestamp,
+                    reasoning=event.step.reason,
+                    actions=event.step.actions,
+                    action_count=len(event.step.actions),
+                    status="running",
+                )
+                self.steps.append(step_data)
+            case ActionEvent():
+                # Update status of corresponding step
+                for step in self.steps:
+                    if step.step_num == event.step_num:
+                        step.status = "error" if event.error else "completed"
+                        break
+            case _:
+                pass

oagi/handler/pyautogui_action_handler.py

@@ -80,6 +80,10 @@ class PyautoguiConfig(BaseModel):
         default="session",
         description="Caps lock handling mode: 'session' (internal state) or 'system' (OS-level)",
     )
+    macos_ctrl_to_cmd: bool = Field(
+        default=True,
+        description="Replace 'ctrl' with 'command' in hotkey combinations on macOS",
+    )
 
 
 class PyautoguiActionHandler:
@@ -169,6 +173,9 @@ class PyautoguiActionHandler:
         # Normalize caps lock variations
         if key in ["caps_lock", "caps", "capslock"]:
             return "capslock"
+        # Remap ctrl to command on macOS if enabled
+        if self.config.macos_ctrl_to_cmd and sys.platform == "darwin" and key == "ctrl":
+            return "command"
         return key
 
     def _parse_hotkey(self, args_str: str) -> list[str]:

oagi/server/agent_wrappers.py

@@ -9,7 +9,7 @@
 import logging
 from typing import TYPE_CHECKING
 
-from ..types import URLImage
+from ..types import URL
 from ..types.models.action import Action
 from .models import ScreenshotRequestData, ScreenshotResponseData
 
@@ -56,7 +56,7 @@ class SocketIOImageProvider:
         self.oagi_client = oagi_client
         self._last_url: str | None = None
 
-    async def __call__(self) -> URLImage:
+    async def __call__(self) -> URL:
         logger.debug("Requesting screenshot via Socket.IO")
 
         # Get S3 presigned URL from OAGI
@@ -87,12 +87,12 @@ class SocketIOImageProvider:
         self.session.current_screenshot_url = upload_response.download_url
 
         logger.debug(f"Screenshot captured successfully: {upload_response.uuid}")
-        return URLImage(upload_response.download_url)
+        return URL(upload_response.download_url)
 
-    async def last_image(self) -> URLImage:
+    async def last_image(self) -> URL:
         if self._last_url:
             logger.debug("Returning last captured screenshot")
-            return URLImage(self._last_url)
+            return URL(self._last_url)
 
         logger.debug("No previous screenshot, capturing new one")
         return await self()

oagi/server/models.py

@@ -15,7 +15,7 @@ from pydantic import BaseModel, Field
 class InitEventData(BaseModel):
     instruction: str = Field(...)
     mode: str | None = Field(default="actor")
-    model: str | None = Field(default="lux-v1")
+    model: str | None = Field(default="lux-actor-1")
     temperature: float | None = Field(default=0.1, ge=0.0, le=2.0)
 
 

oagi/server/session_store.py

@@ -18,7 +18,7 @@ class Session:
         session_id: str,
         instruction: str,
         mode: str = "actor",
-        model: str = "lux-v1",
+        model: str = "lux-actor-1",
         temperature: float = 0.0,
     ):
         self.session_id: str = session_id
@@ -54,7 +54,7 @@ class SessionStore:
         self,
         instruction: str,
         mode: str = "actor",
-        model: str = "lux-v1",
+        model: str = "lux-actor-1",
         temperature: float = 0.0,
         session_id: str | None = None,
     ) -> str: