glaip-sdk 0.7.1__py3-none-any.whl → 0.7.3__py3-none-any.whl

glaip_sdk/agents/base.py

@@ -46,8 +46,6 @@ import inspect
 import logging
 import warnings
 from collections.abc import AsyncGenerator
-
-
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
@@ -942,7 +940,16 @@ class Agent:
 
         if check_local_runtime_available():
             return get_default_runner()
-        raise ValueError(f"{_AGENT_NOT_DEPLOYED_MSG}\n\n{get_local_runtime_missing_message()}")
+
+        # If agent is not deployed, it *must* use local runtime
+        if not self.id:
+            raise ValueError(f"{_AGENT_NOT_DEPLOYED_MSG}\n\n{get_local_runtime_missing_message()}")
+
+        # If agent IS deployed but local execution was forced (local=True)
+        raise ValueError(
+            f"Local execution override was requested, but local runtime is missing.\n\n"
+            f"{get_local_runtime_missing_message()}"
+        )
 
     def _prepare_local_runner_kwargs(
         self,
@@ -977,6 +984,7 @@ class Agent:
         self,
         message: str,
         verbose: bool = False,
+        local: bool = False,
         runtime_config: dict[str, Any] | None = None,
         chat_history: list[dict[str, str]] | None = None,
         **kwargs: Any,
@@ -989,9 +997,13 @@ class Agent:
         - **Local**: When the agent is not deployed and glaip-sdk[local] is installed,
           execution happens locally via aip-agents (no server required).
 
+        You can force local execution for a deployed agent by passing `local=True`.
+
         Args:
             message: The message to send to the agent.
             verbose: If True, print streaming output to console. Defaults to False.
+            local: If True, force local execution even if the agent is deployed.
+                Defaults to False.
             runtime_config: Optional runtime configuration for tools, MCPs, and agents.
                 Keys can be SDK objects, UUIDs, or names. Example:
                 {
@@ -1013,16 +1025,19 @@ class Agent:
             RuntimeError: If server-backed execution fails due to client issues.
         """
         # Backend routing: deployed agents use server, undeployed use local (if available)
-        if self.id:
+        if self.id and not local:
             # Server-backed execution path (agent is deployed)
             agent_client, call_kwargs = self._prepare_run_kwargs(
-                message, verbose, runtime_config or kwargs.get("runtime_config"), **kwargs
+                message,
+                verbose,
+                runtime_config or kwargs.get("runtime_config"),
+                **kwargs,
             )
             if chat_history is not None:
                 call_kwargs["chat_history"] = chat_history
             return agent_client.run_agent(**call_kwargs)
 
-        # Local execution path (agent is not deployed)
+        # Local execution path (agent is not deployed OR local=True)
         runner = self._get_local_runner_or_raise()
         local_kwargs = self._prepare_local_runner_kwargs(message, verbose, runtime_config, chat_history, **kwargs)
         return runner.run(**local_kwargs)
@@ -1031,6 +1046,7 @@ class Agent:
         self,
         message: str,
         verbose: bool = False,
+        local: bool = False,
         runtime_config: dict[str, Any] | None = None,
         chat_history: list[dict[str, str]] | None = None,
         **kwargs: Any,
@@ -1043,9 +1059,13 @@ class Agent:
         - **Local**: When the agent is not deployed and glaip-sdk[local] is installed,
           execution happens locally via aip-agents (no server required).
 
+        You can force local execution for a deployed agent by passing `local=True`.
+
         Args:
             message: The message to send to the agent.
             verbose: If True, print streaming output to console. Defaults to False.
+            local: If True, force local execution even if the agent is deployed.
+                Defaults to False.
             runtime_config: Optional runtime configuration for tools, MCPs, and agents.
                 Keys can be SDK objects, UUIDs, or names. Example:
                 {
@@ -1067,10 +1087,13 @@ class Agent:
             RuntimeError: If server-backed execution fails due to client issues.
         """
         # Backend routing: deployed agents use server, undeployed use local (if available)
-        if self.id:
+        if self.id and not local:
             # Server-backed execution path (agent is deployed)
             agent_client, call_kwargs = self._prepare_run_kwargs(
-                message, verbose, runtime_config or kwargs.get("runtime_config"), **kwargs
+                message,
+                verbose,
+                runtime_config or kwargs.get("runtime_config"),
+                **kwargs,
             )
             if chat_history is not None:
                 call_kwargs["chat_history"] = chat_history
@@ -1079,7 +1102,7 @@ class Agent:
                 yield chunk
             return
 
-        # Local execution path (agent is not deployed)
+        # Local execution path (agent is not deployed OR local=True)
         runner = self._get_local_runner_or_raise()
         local_kwargs = self._prepare_local_runner_kwargs(message, verbose, runtime_config, chat_history, **kwargs)
         result = await runner.arun(**local_kwargs)

glaip_sdk/client/agents.py

@@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any, BinaryIO
 
 if TYPE_CHECKING:
     from glaip_sdk.client.schedules import ScheduleClient
+    from glaip_sdk.hitl.remote import RemoteHITLHandler
 
 import httpx
 from glaip_sdk.agents import Agent
@@ -415,6 +416,7 @@ class AgentClient(BaseClient):
         timeout_seconds: float,
         agent_name: str | None,
         meta: dict[str, Any],
+        hitl_handler: "RemoteHITLHandler | None" = None,
     ) -> tuple[str, dict[str, Any], float | None, float | None]:
         """Process stream events from an HTTP response.
 
@@ -424,6 +426,7 @@ class AgentClient(BaseClient):
             timeout_seconds: Timeout in seconds.
             agent_name: Optional agent name.
             meta: Metadata dictionary.
+            hitl_handler: Optional HITL handler for approval callbacks.
 
         Returns:
             Tuple of (final_text, stats_usage, started_monotonic, finished_monotonic).
@@ -435,6 +438,7 @@ class AgentClient(BaseClient):
             timeout_seconds,
             agent_name,
             meta,
+            hitl_handler=hitl_handler,
         )
 
     def _finalize_renderer(
@@ -1112,6 +1116,7 @@ class AgentClient(BaseClient):
         *,
         renderer: RichStreamRenderer | str | None = "auto",
         runtime_config: dict[str, Any] | None = None,
+        hitl_handler: "RemoteHITLHandler | None" = None,
         **kwargs,
     ) -> str:
         """Run an agent with a message, streaming via a renderer.
@@ -1129,6 +1134,8 @@ class AgentClient(BaseClient):
                     "mcp_configs": {"mcp-id": {"setting": "on"}},
                     "agent_config": {"planning": True},
                 }
+            hitl_handler: Optional RemoteHITLHandler for approval callbacks.
+                Set GLAIP_HITL_AUTO_APPROVE=true for auto-approval without handler.
             **kwargs: Additional arguments to pass to the run API.
 
         Returns:
@@ -1185,6 +1192,7 @@ class AgentClient(BaseClient):
                     timeout_seconds,
                     agent_name,
                     meta,
+                    hitl_handler=hitl_handler,
                 )
 
         except KeyboardInterrupt:
@@ -1201,6 +1209,13 @@ class AgentClient(BaseClient):
             if multipart_data:
                 multipart_data.close()
 
+        # Wait for pending HITL decisions before returning
+        if hitl_handler and hasattr(hitl_handler, "wait_for_pending_decisions"):
+            try:
+                hitl_handler.wait_for_pending_decisions(timeout=30)
+            except Exception as e:
+                logger.warning(f"Error waiting for HITL decisions: {e}")
+
         return self._finalize_renderer(
             r,
             final_text,
@@ -1282,6 +1297,7 @@ class AgentClient(BaseClient):
         *,
         request_timeout: float | None = None,
         runtime_config: dict[str, Any] | None = None,
+        hitl_handler: "RemoteHITLHandler | None" = None,
         **kwargs,
     ) -> AsyncGenerator[dict, None]:
         """Async run an agent with a message, yielding streaming JSON chunks.
@@ -1298,16 +1314,26 @@ class AgentClient(BaseClient):
                     "mcp_configs": {"mcp-id": {"setting": "on"}},
                     "agent_config": {"planning": True},
                 }
+            hitl_handler: Optional HITL handler for remote approval requests.
+                Note: Async HITL support is currently deferred. This parameter
+                is accepted for API consistency but will raise NotImplementedError
+                if provided.
             **kwargs: Additional arguments (chat_history, pii_mapping, etc.)
 
         Yields:
             Dictionary containing parsed JSON chunks from the streaming response
 
         Raises:
+            NotImplementedError: If hitl_handler is provided (async HITL not yet supported)
             AgentTimeoutError: When agent execution times out
             httpx.TimeoutException: When general timeout occurs
             Exception: For other unexpected errors
         """
+        if hitl_handler is not None:
+            raise NotImplementedError(
+                "Async HITL support is currently deferred. "
+                "Please use the synchronous run_agent() method with hitl_handler."
+            )
         # Include runtime_config in kwargs only when caller hasn't already provided it
         if runtime_config is not None and "runtime_config" not in kwargs:
             kwargs["runtime_config"] = runtime_config

glaip_sdk/client/hitl.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+"""HITL REST client for manual approval operations.
+
+Authors:
+    GLAIP SDK Team
+"""
+
+from typing import Any
+
+from glaip_sdk.client.base import BaseClient
+from glaip_sdk.hitl.base import HITLDecision
+
+
+class HITLClient(BaseClient):
+    """Client for HITL REST endpoints.
+
+    Use for manual approval workflows separate from agent runs.
+
+    Example:
+        >>> # List pending approvals
+        >>> pending = client.hitl.list_pending()
+        >>>
+        >>> # Approve a request
+        >>> client.hitl.approve(
+        ...     request_id="bc4d0a77-7800-470e-a91c-7fd663a66b4d",
+        ...     operator_input="Verified and approved",
+        ... )
+    """
+
+    def approve(
+        self,
+        request_id: str,
+        operator_input: str | None = None,
+        run_id: str | None = None,
+    ) -> dict[str, Any]:
+        """Approve a HITL request.
+
+        Args:
+            request_id: HITL request ID from SSE stream
+            operator_input: Optional notes/reason for approval
+            run_id: Optional client-side run correlation ID
+
+        Returns:
+            Response dict: {"status": "ok", "message": "..."}
+        """
+        return self._post_decision(
+            request_id,
+            HITLDecision.APPROVED,
+            operator_input,
+            run_id,
+        )
+
+    def reject(
+        self,
+        request_id: str,
+        operator_input: str | None = None,
+        run_id: str | None = None,
+    ) -> dict[str, Any]:
+        """Reject a HITL request.
+
+        Args:
+            request_id: HITL request ID
+            operator_input: Optional reason for rejection
+            run_id: Optional run correlation ID
+
+        Returns:
+            Response dict
+        """
+        return self._post_decision(
+            request_id,
+            HITLDecision.REJECTED,
+            operator_input,
+            run_id,
+        )
+
+    def skip(
+        self,
+        request_id: str,
+        operator_input: str | None = None,
+        run_id: str | None = None,
+    ) -> dict[str, Any]:
+        """Skip a HITL request.
+
+        Args:
+            request_id: HITL request ID
+            operator_input: Optional notes
+            run_id: Optional run correlation ID
+
+        Returns:
+            Response dict
+        """
+        return self._post_decision(
+            request_id,
+            HITLDecision.SKIPPED,
+            operator_input,
+            run_id,
+        )
+
+    def _post_decision(
+        self,
+        request_id: str,
+        decision: HITLDecision,
+        operator_input: str | None,
+        run_id: str | None,
+    ) -> dict[str, Any]:
+        """Post HITL decision to backend."""
+        payload = {
+            "request_id": request_id,
+            "decision": decision.value,
+        }
+
+        if operator_input:
+            payload["operator_input"] = operator_input
+        if run_id:
+            payload["run_id"] = run_id
+
+        return self._request("POST", "/agents/hitl/decision", json=payload)
+
+    def list_pending(self) -> list[dict[str, Any]]:
+        """List all pending HITL requests.
+
+        Returns:
+            List of pending request dicts with metadata:
+            [
+                {
+                    "request_id": "...",
+                    "tool": "...",
+                    "arguments": {...},
+                    "created_at": "...",
+                    "agent_id": "...",
+                    "hitl_metadata": {...},
+                },
+                ...
+            ]
+        """
+        return self._request("GET", "/agents/hitl/pending")

glaip_sdk/client/main.py

@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any
 
 from glaip_sdk.client.agents import AgentClient
 from glaip_sdk.client.base import BaseClient
+from glaip_sdk.client.hitl import HITLClient
 from glaip_sdk.client.mcps import MCPClient
 from glaip_sdk.client.schedules import ScheduleClient
 from glaip_sdk.client.shared import build_shared_config
@@ -40,6 +41,7 @@ class Client(BaseClient):
         self.tools = ToolClient(**shared_config)
         self.mcps = MCPClient(**shared_config)
         self.schedules = ScheduleClient(**shared_config)
+        self.hitl = HITLClient(**shared_config)
 
     # ---- Core API Methods (Public Interface) ----
 

glaip_sdk/client/run_rendering.py

@@ -11,7 +11,10 @@ import json
 import logging
 from collections.abc import AsyncIterable, Callable
 from time import monotonic
-from typing import Any
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from glaip_sdk.hitl.remote import RemoteHITLHandler
 
 import httpx
 from rich.console import Console as _Console
@@ -130,6 +133,7 @@ class AgentRunRenderingManager:
         timeout_seconds: float,
         agent_name: str | None,
         meta: dict[str, Any],
+        hitl_handler: RemoteHITLHandler | None = None,
     ) -> tuple[str, dict[str, Any], float | None, float | None]:
         """Process streaming events and accumulate response."""
         final_text = ""
@@ -153,6 +157,7 @@ class AgentRunRenderingManager:
                     final_text,
                     stats_usage,
                     meta,
+                    hitl_handler=hitl_handler,
                 )
 
                 if controller and getattr(controller, "enabled", False):
@@ -504,6 +509,7 @@ class AgentRunRenderingManager:
         final_text: str,
         stats_usage: dict[str, Any],
         meta: dict[str, Any],
+        hitl_handler: RemoteHITLHandler | None = None,
     ) -> tuple[str, dict[str, Any]]:
         """Process a single streaming event.
 
@@ -513,6 +519,7 @@ class AgentRunRenderingManager:
             final_text: Accumulated text so far.
             stats_usage: Usage statistics dictionary.
             meta: Metadata dictionary.
+            hitl_handler: Optional HITL handler for approval callbacks.
 
         Returns:
             Tuple of (updated_final_text, updated_stats_usage).
@@ -523,6 +530,17 @@ class AgentRunRenderingManager:
             self._logger.debug("Non-JSON SSE fragment skipped")
             return final_text, stats_usage
 
+        # Handle HITL event (non-blocking via thread)
+        if hitl_handler and self._is_hitl_pending_event(ev):
+            try:
+                hitl_handler.handle_hitl_event(ev)
+            except Exception as e:
+                # Log but don't crash stream
+                self._logger.error(
+                    f"HITL handler error: {e}",
+                    exc_info=True,
+                )
+
         kind = (ev.get("metadata") or {}).get("kind")
         renderer.on_event(ev)
 
@@ -590,6 +608,20 @@ class AgentRunRenderingManager:
             return content
         return final_text
 
+    @staticmethod
+    def _is_hitl_pending_event(event: dict[str, Any]) -> bool:
+        """Check if event is a pending HITL approval request.
+
+        Args:
+            event: Parsed event dictionary.
+
+        Returns:
+            True if event is a pending HITL request.
+        """
+        metadata = event.get("metadata", {})
+        hitl_meta = metadata.get("hitl", {})
+        return hitl_meta.get("required") is True and hitl_meta.get("decision") == "pending"
+
     def _handle_run_info_event(
         self,
         ev: dict[str, Any],

glaip_sdk/hitl/__init__.py

@@ -6,10 +6,23 @@ and remote agent execution modes.
 For local development, LocalPromptHandler is automatically injected when
 agent_config.hitl_enabled is True. No manual setup required.
 
+For remote execution, use RemoteHITLHandler to handle HITL events programmatically.
+
 Authors:
     Putu Ravindra Wiguna (putu.r.wiguna@gdplabs.id)
+    GLAIP SDK Team
 """
 
+from glaip_sdk.hitl.base import HITLCallback, HITLDecision, HITLRequest, HITLResponse
 from glaip_sdk.hitl.local import LocalPromptHandler, PauseResumeCallback
+from glaip_sdk.hitl.remote import RemoteHITLHandler
 
-__all__ = ["LocalPromptHandler", "PauseResumeCallback"]
+__all__ = [
+    "LocalPromptHandler",
+    "PauseResumeCallback",
+    "HITLCallback",
+    "HITLDecision",
+    "HITLRequest",
+    "HITLResponse",
+    "RemoteHITLHandler",
+]

glaip_sdk/hitl/base.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+"""Base types for HITL approval handling.
+
+Authors:
+    GLAIP SDK Team
+"""
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Protocol, runtime_checkable
+
+
+class HITLDecision(str, Enum):
+    """HITL decision types."""
+
+    APPROVED = "approved"
+    REJECTED = "rejected"
+    SKIPPED = "skipped"
+
+
+@dataclass
+class HITLRequest:
+    """HITL approval request from SSE stream."""
+
+    request_id: str
+    tool_name: str
+    tool_args: dict[str, Any]
+    timeout_at: str  # ISO 8601, authoritative deadline
+    timeout_seconds: int  # Informational, fallback only
+
+    # Raw metadata for advanced use cases
+    hitl_metadata: dict[str, Any]
+    tool_metadata: dict[str, Any]
+
+
+@dataclass
+class HITLResponse:
+    """HITL decision response."""
+
+    decision: HITLDecision
+    operator_input: str | None = None
+
+
+@runtime_checkable
+class HITLCallback(Protocol):
+    """Protocol for HITL approval callbacks.
+
+    Callbacks should complete within the computed callback timeout.
+    Callbacks should handle exceptions internally or let them propagate.
+    """
+
+    def __call__(self, request: HITLRequest) -> HITLResponse:
+        """Handle HITL approval request.
+
+        Args:
+            request: HITL request with tool info and metadata
+
+        Returns:
+            HITLResponse with decision and optional operator input
+
+        Raises:
+            Any exception will be caught, logged, and treated as REJECTED.
+        """
+        ...

glaip_sdk/hitl/remote.py

@@ -0,0 +1,523 @@
+#!/usr/bin/env python3
+"""Remote HITL approval handler with threading and error recovery.
+
+Authors:
+    GLAIP SDK Team
+"""
+
+import logging
+import os
+import threading
+import time
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Any
+from collections.abc import Callable
+
+import httpx
+
+from glaip_sdk.exceptions import APIError
+from glaip_sdk.hitl.base import (
+    HITLCallback,
+    HITLDecision,
+    HITLRequest,
+    HITLResponse,
+)
+
+if TYPE_CHECKING:
+    from glaip_sdk.client.base import BaseClient
+
+logger = logging.getLogger(__name__)
+
+
+class RemoteHITLHandler:
+    """Handler for remote HITL approval requests.
+
+    Executes callbacks in background threads to avoid blocking SSE stream.
+    Includes timeout enforcement and error handling.
+
+    Thread Safety:
+        This handler is thread-safe for concurrent HITL events. Callbacks
+        execute in daemon threads. Use wait_for_pending_decisions() after
+        stream completion to ensure all decisions are posted.
+
+    Environment Variables:
+        GLAIP_HITL_AUTO_APPROVE: Set to "true" to auto-approve all requests.
+
+    Example:
+        >>> def my_approver(request: HITLRequest) -> HITLResponse:
+        ...     print(f"Approve {request.tool_name}?")
+        ...     return HITLResponse(decision=HITLDecision.APPROVED)
+        >>>
+        >>> handler = RemoteHITLHandler(callback=my_approver, client=client)
+        >>> client.agents.run_agent(agent_id, message, hitl_handler=handler)
+    """
+
+    def __init__(
+        self,
+        callback: HITLCallback | None = None,
+        *,
+        client: "BaseClient",
+        auto_approve: bool | None = None,
+        max_retries: int = 3,
+        on_unrecoverable_error: Callable[[str, Exception], None] | None = None,
+    ):
+        """Initialize remote HITL handler.
+
+        Args:
+            callback: Function to invoke for approval decisions.
+                If None and auto_approve=False, HITL events will be rejected.
+            client: BaseClient instance for posting decisions.
+            auto_approve: Override GLAIP_HITL_AUTO_APPROVE env var.
+            max_retries: Max retries for POST /agents/hitl/decision (default: 3)
+            on_unrecoverable_error: Optional callback invoked when both the
+                approval callback and fallback rejection POST fail. Receives
+                (request_id, exception) for custom alerting/logging.
+        """
+        self._callback = callback
+        self._client = client
+        self._max_retries = max_retries
+        self._on_unrecoverable_error = on_unrecoverable_error
+
+        # Thread tracking for synchronization
+        self._active_threads: list[threading.Thread] = []
+        self._threads_lock = threading.Lock()
+
+        # Auto-approve from env or explicit override
+        if auto_approve is None:
+            auto_approve = os.getenv("GLAIP_HITL_AUTO_APPROVE", "").lower() == "true"
+        self._auto_approve = auto_approve
+
+        # Warn if no decision mechanism
+        if not auto_approve and callback is None:
+            logger.warning(
+                "RemoteHITLHandler: No callback provided and auto_approve=False. "
+                "HITL requests will be rejected. Set GLAIP_HITL_AUTO_APPROVE=true "
+                "or provide a callback."
+            )
+
+    def handle_hitl_event(self, event: dict) -> None:
+        """Process HITL event from SSE stream.
+
+        Runs in background thread to avoid blocking stream.
+
+        Args:
+            event: SSE event dict with metadata.hitl and metadata.tool_info
+        """
+        # Validate event structure
+        try:
+            request = self._parse_hitl_request(event)
+        except (KeyError, ValueError) as e:
+            logger.error(f"Invalid HITL event structure: {e}")
+            logger.debug(f"Event data: {event}")
+            return
+
+        # Execute in background thread to avoid blocking stream
+        thread = threading.Thread(
+            target=self._process_approval,
+            args=(request,),
+            daemon=True,
+            name=f"hitl-{request.request_id[:8]}",
+        )
+        thread.start()
+
+        # Track active threads for synchronization.
+        cleanup_snapshot: list[threading.Thread] | None = None
+        with self._threads_lock:
+            self._active_threads.append(thread)
+            if len(self._active_threads) > 10:
+                cleanup_snapshot = list(self._active_threads)
+
+        # Clean up finished threads outside the lock when the list grows.
+        if cleanup_snapshot is not None:
+            dead_threads = [t for t in cleanup_snapshot if not t.is_alive()]
+            if dead_threads:
+                with self._threads_lock:
+                    self._active_threads = [t for t in self._active_threads if t not in dead_threads]
+
+    def _parse_hitl_request(self, event: dict) -> HITLRequest:
+        """Parse SSE event into HITLRequest.
+
+        Raises:
+            KeyError: If required fields missing
+            ValueError: If data invalid
+        """
+        metadata = event.get("metadata", {})
+        hitl_meta = metadata.get("hitl", {})
+        tool_info = metadata.get("tool_info", {})
+
+        # Validate required fields
+        request_id = hitl_meta.get("request_id")
+        if not request_id:
+            raise ValueError("Missing request_id in HITL metadata")
+
+        return HITLRequest(
+            request_id=request_id,
+            tool_name=tool_info.get("name", "unknown"),
+            tool_args=tool_info.get("args", {}),
+            timeout_at=hitl_meta.get("timeout_at", ""),
+            timeout_seconds=hitl_meta.get("timeout_seconds", 180),
+            hitl_metadata=hitl_meta,
+            tool_metadata=tool_info,
+        )
+
+    def _process_approval(self, request: HITLRequest) -> None:
+        """Process approval in background thread.
+
+        Handles callback execution, timeout, errors, and POST retry.
+        """
+        try:
+            # Get decision
+            response = self._get_decision(request)
+
+            # Post to backend with retry
+            self._post_decision_with_retry(request.request_id, response)
+
+        except APIError as e:
+            # Handle client errors (4xx) - non-retryable
+            if e.status_code and 400 <= e.status_code < 500:
+                logger.warning(f"Non-retryable HITL decision error for {request.request_id}: {e}")
+                return
+
+            logger.error(
+                f"HITL processing failed for {request.request_id}: {e}",
+                exc_info=True,
+            )
+            self._handle_approval_failure(request, e)
+
+        except Exception as e:
+            logger.error(
+                f"HITL processing failed for {request.request_id}: {e}",
+                exc_info=True,
+            )
+            self._handle_approval_failure(request, e)
+
+    def _handle_approval_failure(self, request: HITLRequest, error: Exception) -> None:
+        """Handle failure during approval processing by attempting fallback rejection.
+
+        Args:
+            request: The HITL request that failed.
+            error: The exception that occurred during processing.
+        """
+        # Try to post rejection as fallback
+        try:
+            fallback = HITLResponse(
+                decision=HITLDecision.REJECTED,
+                operator_input=f"Error: {str(error)[:100]}",
+            )
+            self._post_decision_with_retry(request.request_id, fallback)
+        except Exception as post_err:
+            logger.error(f"Failed to post fallback rejection: {post_err}")
+
+            # Invoke error callback if provided
+            if self._on_unrecoverable_error:
+                try:
+                    self._on_unrecoverable_error(request.request_id, post_err)
+                except Exception as cb_err:
+                    logger.error(f"Error callback failed: {cb_err}")
+
+    def _get_decision(self, request: HITLRequest) -> HITLResponse:
+        """Get approval decision via auto-approve or callback.
+
+        Raises:
+            Exception: If callback fails
+        """
+        # Auto-approve path
+        if self._auto_approve:
+            logger.info(f"Auto-approving HITL request {request.request_id}")
+            return HITLResponse(
+                decision=HITLDecision.APPROVED,
+                operator_input="auto-approved",
+            )
+
+        # Callback path
+        if self._callback:
+            return self._execute_callback(request)
+
+        # No callback, no auto-approve -> reject
+        logger.warning(f"No approval mechanism, rejecting {request.request_id}")
+        return HITLResponse(
+            decision=HITLDecision.REJECTED,
+            operator_input="No approval handler configured",
+        )
+
+    def _execute_callback(self, request: HITLRequest) -> HITLResponse:
+        """Execute callback with timeout and error handling.
+
+        Args:
+            request: HITL request to process
+
+        Returns:
+            HITLResponse from callback or rejection on error
+        """
+        try:
+            # Apply timeout: 80% of remaining backend time (20% buffer)
+            timeout = self._compute_callback_timeout(request)
+
+            # Run callback with timeout
+            response = self._run_callback_with_timeout(
+                request,
+                timeout_seconds=timeout,
+            )
+
+            # Validate return type
+            if not isinstance(response, HITLResponse):
+                logger.error(
+                    f"HITL callback returned invalid type {type(response)} "
+                    f"for {request.request_id}, expected HITLResponse"
+                )
+                return HITLResponse(
+                    decision=HITLDecision.REJECTED,
+                    operator_input="Callback returned invalid response type",
+                )
+
+            logger.info(f"HITL callback returned {response.decision} for {request.request_id}")
+            return response
+
+        except TimeoutError:
+            logger.error(
+                f"HITL callback timeout ({timeout}s) for request {request.request_id} (tool: {request.tool_name})"
+            )
+            return HITLResponse(
+                decision=HITLDecision.REJECTED,
+                operator_input="Callback timeout",
+            )
+        except Exception as e:
+            logger.error(
+                f"HITL callback failed for {request.request_id}: {e}",
+                exc_info=True,
+            )
+            return HITLResponse(
+                decision=HITLDecision.REJECTED,
+                operator_input=f"Callback error: {str(e)[:100]}",
+            )
+
+    def _run_callback_with_timeout(
+        self,
+        request: HITLRequest,
+        timeout_seconds: int,
+    ) -> HITLResponse:
+        """Run callback with timeout using threading.
+
+        Args:
+            request: HITL request
+            timeout_seconds: Max execution time
+
+        Returns:
+            HITLResponse from callback
+
+        Raises:
+            TimeoutError: If callback exceeds timeout
+            Exception: If callback raises
+        """
+        result = [None]  # Mutable container for thread result
+        exception = [None]
+
+        def wrapper():
+            try:
+                result[0] = self._callback(request)
+            except Exception as e:
+                exception[0] = e
+
+        thread = threading.Thread(target=wrapper, daemon=True)
+        thread.start()
+        thread.join(timeout=timeout_seconds)
+
+        if thread.is_alive():
+            # Timeout - thread still running
+            logger.warning(f"Callback timeout after {timeout_seconds}s for {request.request_id}")
+            raise TimeoutError(f"Callback exceeded {timeout_seconds}s")
+
+        if exception[0]:
+            raise exception[0]
+
+        if result[0] is None:
+            raise ValueError("Callback returned None instead of HITLResponse")
+
+        return result[0]
+
+    def _compute_callback_timeout(self, request: HITLRequest) -> int:
+        """Compute callback timeout using timeout_at as the source of truth.
+
+        Args:
+            request: HITL request with timeout information
+
+        Returns:
+            Timeout in seconds (minimum 5s)
+        """
+        fallback_seconds = max(5, int(request.timeout_seconds * 0.8))
+        try:
+            # Try ISO format first with Z suffix
+            if request.timeout_at.endswith("Z"):
+                deadline = datetime.fromisoformat(request.timeout_at.replace("Z", "+00:00"))
+            else:
+                # Try parsing as-is (may include timezone info)
+                deadline = datetime.fromisoformat(request.timeout_at)
+
+            now = datetime.now(timezone.utc)
+            remaining = max(0, int((deadline - now).total_seconds()))
+            return max(5, int(remaining * 0.8))
+        except (TypeError, ValueError, AttributeError) as e:
+            logger.debug(
+                f"Failed to parse timeout_at '{request.timeout_at}': {e}, using fallback timeout of {fallback_seconds}s"
+            )
+            return fallback_seconds
+
+    def _post_decision_with_retry(
+        self,
+        request_id: str,
+        response: HITLResponse,
+    ) -> None:
+        """Post decision to backend with retry logic.
+
+        Only retries on server errors (5xx) and network errors.
+        Client errors (4xx) fail immediately as they won't succeed on retry.
+        404/409 are treated as already resolved.
+
+        Args:
+            request_id: HITL request ID
+            response: Decision response
+
+        Raises:
+            Exception: If all retries fail
+        """
+        payload = self._build_decision_payload(request_id, response)
+        last_error = None
+
+        for attempt in range(1, self._max_retries + 1):
+            try:
+                result = self._client._request("POST", "/agents/hitl/decision", json=payload)
+                logger.info(
+                    f"HITL decision posted successfully for {request_id} (attempt {attempt}/{self._max_retries})"
+                )
+                logger.debug(f"Response: {result}")
+                return  # Success
+
+            except APIError as e:
+                last_error = e
+                if self._handle_api_error(e, request_id, attempt):
+                    return  # Request already resolved
+
+            except httpx.RequestError as e:
+                last_error = e
+                logger.warning(f"Network error (attempt {attempt}/{self._max_retries}): {e}")
+
+            except Exception as e:
+                # Unexpected errors - don't retry
+                logger.error(f"Unexpected error posting decision: {e}")
+                raise
+
+            # Retry delay only if not last attempt
+            if attempt < self._max_retries:
+                time.sleep(attempt)  # Linear backoff: 1s, 2s, 3s
+
+        # All retries failed
+        self._log_retry_exhausted(request_id, last_error)
+        raise last_error
+
+    def _build_decision_payload(self, request_id: str, response: HITLResponse) -> dict[str, Any]:
+        """Build payload for decision POST request."""
+        payload = {
+            "request_id": request_id,
+            "decision": response.decision.value,
+        }
+
+        if response.operator_input:
+            payload["operator_input"] = response.operator_input
+
+        return payload
+
+    def _handle_api_error(self, error: APIError, request_id: str, attempt: int) -> bool:
+        """Handle API error and determine if request is already resolved.
+
+        Args:
+            error: The API error to handle
+            request_id: The HITL request ID
+            attempt: Current attempt number
+
+        Returns:
+            True if request is already resolved (404/409), False otherwise
+
+        Raises:
+            APIError: If error is not retryable
+        """
+        status_code = error.status_code or 0
+        RETRYABLE_STATUS_CODES = {500, 502, 503, 504}
+
+        # 404/409 indicate the request is already resolved
+        if status_code in (404, 409):
+            logger.info(f"Request already resolved ({status_code}) for {request_id}")
+            return True
+
+        # Don't retry client errors (4xx)
+        if 400 <= status_code < 500:
+            logger.warning(f"Non-retryable error {status_code} for {request_id}: {error}")
+            raise error
+
+        # Retry server errors (5xx)
+        if status_code not in RETRYABLE_STATUS_CODES:
+            logger.warning(f"Unexpected status {status_code}, not retrying")
+            raise error
+
+        logger.warning(f"Server error {status_code} (attempt {attempt}/{self._max_retries}): {error}")
+        return False
+
+    def _log_retry_exhausted(self, request_id: str, last_error: Exception | None) -> None:
+        """Log that retry attempts have been exhausted."""
+        logger.error(f"Failed to post HITL decision for {request_id} after {self._max_retries} attempts: {last_error}")
+
+    def wait_for_pending_decisions(self, timeout: float = 30) -> None:
+        """Wait for all pending HITL decision posts to complete.
+
+        Call this after SSE stream ends to ensure all background
+        threads finish posting decisions before returning from run_agent().
+
+        Uses adaptive timeout redistribution: when threads complete early,
+        their remaining time is redistributed to remaining threads.
+
+        Args:
+            timeout: Maximum seconds to wait for all threads (default: 30)
+
+        Raises:
+            ValueError: If timeout is not positive
+        """
+        if timeout <= 0:
+            raise ValueError("timeout must be positive")
+
+        with self._threads_lock:
+            threads_to_wait = self._active_threads.copy()
+
+        if not threads_to_wait:
+            return
+
+        deadline = time.monotonic() + timeout
+        remaining_threads = list(threads_to_wait)
+
+        while remaining_threads:
+            time_left = deadline - time.monotonic()
+            if time_left <= 0:
+                break
+
+            per_thread_timeout = time_left / len(remaining_threads)
+            next_round: list[threading.Thread] = []
+
+            for thread in remaining_threads:
+                thread.join(timeout=per_thread_timeout)
+                if thread.is_alive():
+                    logger.warning(f"HITL thread {thread.name} still running after {per_thread_timeout:.1f}s timeout")
+                    next_round.append(thread)
+
+            if len(next_round) == len(remaining_threads):
+                # Break to avoid a tight loop when joins return immediately.
+                break
+
+            remaining_threads = next_round
+
+        # Clean up finished threads
+        with self._threads_lock:
+            still_alive = [t for t in self._active_threads if t.is_alive()]
+            if still_alive:
+                logger.error(
+                    f"{len(still_alive)} HITL threads did not complete within timeout. "
+                    "Decisions may not have been posted."
+                )
+            self._active_threads = still_alive

{glaip_sdk-0.7.1.dist-info → glaip_sdk-0.7.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: glaip-sdk
-Version: 0.7.1
+Version: 0.7.3
 Summary: Python SDK and CLI for GL AIP (GDP Labs AI Agent Package) - Build, run, and manage AI agents
 Author-email: Raymond Christopher <raymond.christopher@gdplabs.id>
 License: MIT

{glaip_sdk-0.7.1.dist-info → glaip_sdk-0.7.3.dist-info}/RECORD

@@ -5,7 +5,7 @@ glaip_sdk/exceptions.py,sha256=iAChFClkytXRBLP0vZq1_YjoZxA9i4m4bW1gDLiGR1g,2321
 glaip_sdk/icons.py,sha256=J5THz0ReAmDwIiIooh1_G3Le-mwTJyEjhJDdJ13KRxM,524
 glaip_sdk/rich_components.py,sha256=44Z0V1ZQleVh9gUDGwRR5mriiYFnVGOhm7fFxZYbP8c,4052
 glaip_sdk/agents/__init__.py,sha256=VfYov56edbWuySXFEbWJ_jLXgwnFzPk1KB-9-mfsUCc,776
-glaip_sdk/agents/base.py,sha256=D3Ll2l_uxv2kIwPCuDhfy_bbSxHrHPaxp6op2c7TBl0,43580
+glaip_sdk/agents/base.py,sha256=GQnzCw2cqlrbxwdoWFfhBcBlEDgubY4tlD6gr1b3zps,44539
 glaip_sdk/cli/__init__.py,sha256=xCCfuF1Yc7mpCDcfhHZTX0vizvtrDSLeT8MJ3V7m5A0,156
 glaip_sdk/cli/account_store.py,sha256=TK4iTV93Q1uD9mCY_2ZMT6EazHKU2jX0qhgWfEM4V-4,18459
 glaip_sdk/cli/agent_config.py,sha256=YAbFKrTNTRqNA6b0i0Q3pH-01rhHDRi5v8dxSFwGSwM,2401
@@ -102,11 +102,12 @@ glaip_sdk/cli/transcript/viewer.py,sha256=Y4G40WR6v1g4TfxRbGSZqdrqhLcqBxoWkQgToQ
 glaip_sdk/client/__init__.py,sha256=s2REOumgE8Z8lA9dWJpwXqpgMdzSELSuCQkZ7sYngX0,381
 glaip_sdk/client/_schedule_payloads.py,sha256=9BXa75CCx3clsKgwmG9AWyvhPY6kVwzQtoLvTTw40CQ,2759
 glaip_sdk/client/agent_runs.py,sha256=tZSFEZZ3Yx0uYRgnwkLe-X0TlmgKJQ-ivzb6SrVnxY8,4862
-glaip_sdk/client/agents.py,sha256=jR6DmpW1XRvpa_ewtXfzOH-q2ipTIF--yHZDdasq5Mk,48249
+glaip_sdk/client/agents.py,sha256=kUNklI6QS39dgPFWMy1F0uw8En4rZ2fDiPW9B87jaVU,49724
 glaip_sdk/client/base.py,sha256=BhNaC2TJJ2jVWRTYmfxD3WjYgAyIuWNz9YURdNXXjJo,18245
-glaip_sdk/client/main.py,sha256=hhMw2c64BfxYavvA2Hz0xCTQPe7EcOXjGKIahwsX_DI,9269
+glaip_sdk/client/hitl.py,sha256=dO_q-43miI0oGrJDyUrZ9MbettQp0hai4kjvPaYm010,3545
+glaip_sdk/client/main.py,sha256=FqN-JIoCTN1pV-3O-ElIbe8t_cWYUDV4S0N8W018De0,9362
 glaip_sdk/client/mcps.py,sha256=-JdaIkg0QE3egJ8p93eoOPULup8KbM2WRCcwlvqlqrA,14492
-glaip_sdk/client/run_rendering.py,sha256=kERp78v50jojsNWHrjNEkbC8sgOpMacaqUdw5YZuK6A,26074
+glaip_sdk/client/run_rendering.py,sha256=L1mPDPQW_NLs7yi6S1dtJlnTUWVl9R9ioxVzcC8xBrU,27262
 glaip_sdk/client/schedules.py,sha256=ZfPzCYzk4YRuPkjkTTgLe5Rqa07mi-h2WmP4H91mMZ0,14113
 glaip_sdk/client/shared.py,sha256=esHlsR0LEfL-pFDaWebQjKKOLl09jsRY-2pllBUn4nU,522
 glaip_sdk/client/tools.py,sha256=NzQTIsn-bjYN9EfGWCBqqawCIVs7auaccFv7BM_3oCc,23871
@@ -115,8 +116,10 @@ glaip_sdk/client/payloads/agent/__init__.py,sha256=gItEH2zt2secVq6n60oGA-ztdE5mc
 glaip_sdk/client/payloads/agent/requests.py,sha256=5FuGEuypaEXlWBhB07JrDca_ecLg4bvo8mjyFBxAV9U,17139
 glaip_sdk/client/payloads/agent/responses.py,sha256=1eRMI4JAIGqTB5zY_7D9ILQDRHPXR06U7JqHSmRp3Qs,1243
 glaip_sdk/config/constants.py,sha256=Y03c6op0e7K0jTQ8bmWXhWAqsnjWxkAhWniq8Z0iEKY,1081
-glaip_sdk/hitl/__init__.py,sha256=sg92Rpu8_vJIGi1ZEhx0-qWa1nGdvfrKyJAxtoDSKzo,494
+glaip_sdk/hitl/__init__.py,sha256=PHayqIUL9BOh9QZQilisThisPL3G7ylG-XaPdllC_zc,851
+glaip_sdk/hitl/base.py,sha256=EUN2igzydlYZ6_qmHU46Gyk3Bk9uyalZkCJ06XMRKJ8,1484
 glaip_sdk/hitl/local.py,sha256=rzmaRK15BxgRX7cmklUcGQUotMYg8x2Gd9BWf39k6hw,5661
+glaip_sdk/hitl/remote.py,sha256=cdO-wWwRGdyb0HYNMwIvHfvKwOqhqp-l7efnaC9b85M,18914
 glaip_sdk/mcps/__init__.py,sha256=4jYrt8K__oxrxexHRcmnRBXt-W_tbJN61H9Kf2lVh4Q,551
 glaip_sdk/mcps/base.py,sha256=jWwHjDF67_mtDGRp9p5SolANjVeB8jt1PSwPBtX876M,11654
 glaip_sdk/models/__init__.py,sha256=Yfy5CKe9xt3MkyoRIiZ4PYy19LbEfv3GXi2PgqXW7iU,2973
@@ -200,8 +203,8 @@ glaip_sdk/utils/rendering/steps/format.py,sha256=Chnq7OBaj8XMeBntSBxrX5zSmrYeGcO
 glaip_sdk/utils/rendering/steps/manager.py,sha256=BiBmTeQMQhjRMykgICXsXNYh1hGsss-fH9BIGVMWFi0,13194
 glaip_sdk/utils/rendering/viewer/__init__.py,sha256=XrxmE2cMAozqrzo1jtDFm8HqNtvDcYi2mAhXLXn5CjI,457
 glaip_sdk/utils/rendering/viewer/presenter.py,sha256=mlLMTjnyeyPVtsyrAbz1BJu9lFGQSlS-voZ-_Cuugv0,5725
-glaip_sdk-0.7.1.dist-info/METADATA,sha256=Pb8KaBV3ypCfgVvXTZd8mWSP_MgdaqFFdUTEmgPal5I,8365
-glaip_sdk-0.7.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-glaip_sdk-0.7.1.dist-info/entry_points.txt,sha256=65vNPUggyYnVGhuw7RhNJ8Fp2jygTcX0yxJBcBY3iLU,48
-glaip_sdk-0.7.1.dist-info/top_level.txt,sha256=td7yXttiYX2s94-4wFhv-5KdT0rSZ-pnJRSire341hw,10
-glaip_sdk-0.7.1.dist-info/RECORD,,
+glaip_sdk-0.7.3.dist-info/METADATA,sha256=9tLJ5ibX4VkSGKji1SulcZiJ58d6Jefg8ts_UaLdU_E,8365
+glaip_sdk-0.7.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+glaip_sdk-0.7.3.dist-info/entry_points.txt,sha256=65vNPUggyYnVGhuw7RhNJ8Fp2jygTcX0yxJBcBY3iLU,48
+glaip_sdk-0.7.3.dist-info/top_level.txt,sha256=td7yXttiYX2s94-4wFhv-5KdT0rSZ-pnJRSire341hw,10
+glaip_sdk-0.7.3.dist-info/RECORD,,