aury-agent 0.0.4__py3-none-any.whl

aury/agents/eval/__init__.py

@@ -0,0 +1,331 @@
+"""Evaluation framework for agent testing.
+
+TODO: Implement evaluation suite.
+
+This module will provide:
+- EvalSuite: Define and run evaluation test suites
+- TestCase: Individual test case definition
+- EvalResult: Test result with metrics
+- Evaluators: Built-in evaluators (exact match, semantic, etc.)
+
+Reference: Agent evaluation best practices
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Callable, Awaitable, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ..core.base import BaseAgent
+
+
+# =============================================================================
+# Test Case
+# =============================================================================
+
+@dataclass
+class TestCase:
+    """A single evaluation test case.
+    
+    TODO: Implement test case execution.
+    
+    Usage:
+        case = TestCase(
+            name="simple_math",
+            input="What is 2 + 2?",
+            expected="4",
+            evaluator="contains",
+            tags=["math", "basic"],
+        )
+    """
+    name: str
+    input: str | dict[str, Any]
+    expected: str | dict[str, Any] | None = None
+    evaluator: str = "exact"  # "exact", "contains", "semantic", "custom"
+    custom_evaluator: Callable[[str, str], Awaitable[float]] | None = None
+    tags: list[str] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+    timeout: float = 30.0
+
+
+# =============================================================================
+# Eval Result
+# =============================================================================
+
+class EvalStatus(Enum):
+    """Evaluation status."""
+    PASS = "pass"
+    FAIL = "fail"
+    ERROR = "error"
+    SKIP = "skip"
+
+
+@dataclass
+class EvalResult:
+    """Result of a single test case.
+    
+    TODO: Implement result tracking.
+    """
+    test_name: str
+    status: EvalStatus
+    score: float  # 0.0 - 1.0
+    actual_output: str | None = None
+    expected_output: str | None = None
+    error_message: str | None = None
+    duration_ms: float = 0.0
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class SuiteResult:
+    """Result of an entire test suite.
+    
+    TODO: Implement suite-level metrics.
+    """
+    suite_name: str
+    results: list[EvalResult] = field(default_factory=list)
+    total_duration_ms: float = 0.0
+    
+    @property
+    def total(self) -> int:
+        return len(self.results)
+    
+    @property
+    def passed(self) -> int:
+        return sum(1 for r in self.results if r.status == EvalStatus.PASS)
+    
+    @property
+    def failed(self) -> int:
+        return sum(1 for r in self.results if r.status == EvalStatus.FAIL)
+    
+    @property
+    def errors(self) -> int:
+        return sum(1 for r in self.results if r.status == EvalStatus.ERROR)
+    
+    @property
+    def pass_rate(self) -> float:
+        return self.passed / self.total if self.total > 0 else 0.0
+    
+    @property
+    def avg_score(self) -> float:
+        scores = [r.score for r in self.results]
+        return sum(scores) / len(scores) if scores else 0.0
+
+
+# =============================================================================
+# TODO: Evaluators
+# =============================================================================
+
+class Evaluator:
+    """Base evaluator interface.
+    
+    TODO: Implement evaluators.
+    """
+    
+    async def evaluate(
+        self,
+        actual: str,
+        expected: str | None,
+        context: dict[str, Any] | None = None,
+    ) -> float:
+        """Evaluate output against expected.
+        
+        Returns:
+            Score from 0.0 to 1.0
+        """
+        raise NotImplementedError("TODO: Evaluator not yet implemented")
+
+
+class ExactMatchEvaluator(Evaluator):
+    """Exact string match evaluator.
+    
+    TODO: Implement exact match.
+    """
+    
+    async def evaluate(
+        self,
+        actual: str,
+        expected: str | None,
+        context: dict[str, Any] | None = None,
+    ) -> float:
+        raise NotImplementedError("TODO: ExactMatchEvaluator not yet implemented")
+
+
+class ContainsEvaluator(Evaluator):
+    """Check if expected is contained in actual.
+    
+    TODO: Implement contains check.
+    """
+    
+    async def evaluate(
+        self,
+        actual: str,
+        expected: str | None,
+        context: dict[str, Any] | None = None,
+    ) -> float:
+        raise NotImplementedError("TODO: ContainsEvaluator not yet implemented")
+
+
+class SemanticEvaluator(Evaluator):
+    """Semantic similarity evaluator using embeddings.
+    
+    TODO: Implement semantic evaluation.
+    """
+    
+    def __init__(self, threshold: float = 0.8):
+        self.threshold = threshold
+    
+    async def evaluate(
+        self,
+        actual: str,
+        expected: str | None,
+        context: dict[str, Any] | None = None,
+    ) -> float:
+        raise NotImplementedError("TODO: SemanticEvaluator not yet implemented")
+
+
+class LLMJudgeEvaluator(Evaluator):
+    """Use LLM as judge for evaluation.
+    
+    TODO: Implement LLM judge.
+    """
+    
+    def __init__(self, criteria: str | None = None):
+        self.criteria = criteria
+    
+    async def evaluate(
+        self,
+        actual: str,
+        expected: str | None,
+        context: dict[str, Any] | None = None,
+    ) -> float:
+        raise NotImplementedError("TODO: LLMJudgeEvaluator not yet implemented")
+
+
+# =============================================================================
+# TODO: EvalSuite
+# =============================================================================
+
+class EvalSuite:
+    """Evaluation test suite.
+    
+    TODO: Implement evaluation suite.
+    
+    Usage:
+        suite = EvalSuite(
+            name="math_tests",
+            agent=my_agent,
+            cases=[
+                TestCase(name="add", input="2+2?", expected="4"),
+                TestCase(name="mul", input="3*4?", expected="12"),
+            ],
+        )
+        
+        result = await suite.run()
+        print(f"Pass rate: {result.pass_rate:.1%}")
+    """
+    
+    def __init__(
+        self,
+        name: str,
+        agent: "BaseAgent",
+        cases: list[TestCase],
+        evaluators: dict[str, Evaluator] | None = None,
+        parallel: bool = False,
+        max_workers: int = 4,
+    ):
+        self.name = name
+        self.agent = agent
+        self.cases = cases
+        self.evaluators = evaluators or {
+            "exact": ExactMatchEvaluator(),
+            "contains": ContainsEvaluator(),
+            "semantic": SemanticEvaluator(),
+            "llm_judge": LLMJudgeEvaluator(),
+        }
+        self.parallel = parallel
+        self.max_workers = max_workers
+        raise NotImplementedError("TODO: EvalSuite not yet implemented")
+    
+    async def run(
+        self,
+        tags: list[str] | None = None,
+        verbose: bool = False,
+    ) -> SuiteResult:
+        """Run all test cases.
+        
+        Args:
+            tags: Only run cases with these tags (None = all)
+            verbose: Print progress
+            
+        Returns:
+            Suite result with all test results
+        """
+        raise NotImplementedError("TODO: EvalSuite.run not yet implemented")
+    
+    async def run_case(self, case: TestCase) -> EvalResult:
+        """Run a single test case.
+        
+        Args:
+            case: Test case to run
+            
+        Returns:
+            Evaluation result
+        """
+        raise NotImplementedError("TODO: EvalSuite.run_case not yet implemented")
+    
+    def add_case(self, case: TestCase) -> None:
+        """Add a test case to the suite."""
+        self.cases.append(case)
+    
+    def filter_by_tags(self, tags: list[str]) -> list[TestCase]:
+        """Filter cases by tags."""
+        return [c for c in self.cases if any(t in c.tags for t in tags)]
+
+
+# =============================================================================
+# Helper functions
+# =============================================================================
+
+def test_case(
+    name: str,
+    input: str | dict[str, Any],
+    expected: str | dict[str, Any] | None = None,
+    evaluator: str = "exact",
+    **kwargs: Any,
+) -> TestCase:
+    """Convenience function to create test cases.
+    
+    Usage:
+        cases = [
+            test_case("simple", "2+2?", "4"),
+            test_case("semantic", "hello", "hi there", evaluator="semantic"),
+        ]
+    """
+    return TestCase(
+        name=name,
+        input=input,
+        expected=expected,
+        evaluator=evaluator,
+        **kwargs,
+    )
+
+
+__all__ = [
+    # Test case
+    "TestCase",
+    "test_case",
+    # Results
+    "EvalStatus",
+    "EvalResult",
+    "SuiteResult",
+    # Evaluators
+    "Evaluator",
+    "ExactMatchEvaluator",
+    "ContainsEvaluator",
+    "SemanticEvaluator",
+    "LLMJudgeEvaluator",
+    # Suite
+    "EvalSuite",
+]

aury/agents/hitl/__init__.py

@@ -0,0 +1,57 @@
+"""HITL (Human-in-the-Loop) components."""
+from .compaction import (
+    SessionCompaction,
+    CompactionConfig,
+)
+from .revert import (
+    SessionRevert,
+    RevertState,
+    BlockBackend,
+)
+from .exceptions import (
+    SuspendSignal,
+    HITLSuspend,
+    HITLTimeoutError,
+    HITLCancelledError,
+    HITLRequest,
+)
+from .ask_user import (
+    AskUserTool,
+    ConfirmTool,
+)
+from .permission import (
+    Permission,
+    PermissionRules,
+    PermissionSpec,
+    RejectedError,
+    SkippedError,
+    HumanResponse,
+)
+
+__all__ = [
+    # Compaction
+    "SessionCompaction",
+    "CompactionConfig",
+    # Revert
+    "SessionRevert",
+    "RevertState",
+    "BlockBackend",
+    # Signals
+    "SuspendSignal",
+    "HITLSuspend",
+    # Exceptions
+    "HITLTimeoutError",
+    "HITLCancelledError",
+    # Types
+    "HITLRequest",
+    # Tools
+    "AskUserTool",
+    "ConfirmTool",
+    # Permission
+    "Permission",
+    "PermissionRules",
+    "PermissionSpec",
+    "RejectedError",
+    "SkippedError",
+    "HumanResponse",
+]

aury/agents/hitl/ask_user.py

@@ -0,0 +1,242 @@
+"""AskUser tool for LLM to request human input.
+
+This tool allows the LLM to pause execution and ask the user
+for clarification or additional information.
+"""
+from __future__ import annotations
+
+from typing import Any, TYPE_CHECKING
+
+from ..tool import BaseTool, ToolResult
+from ..core.types.session import generate_id
+from ..core.types.block import BlockEvent
+from ..core.signals import HITLSuspend
+from .exceptions import HITLRequest
+
+if TYPE_CHECKING:
+    from ..core.context import InvocationContext
+
+
+class AskUserTool(BaseTool):
+    """Tool for LLM to ask user questions.
+    
+    When executed, this tool:
+    1. Checkpoints current state
+    2. Updates invocation status to SUSPENDED
+    3. Emits a hitl_request block to frontend
+    4. Raises HITLSuspend signal to pause execution
+    
+    The user's response comes via:
+    - agent.respond(request_id, response)
+    - agent.run(response) (auto-detects suspended state)
+    
+    Note: Uses HITLSuspend (inherits BaseException) so it won't be caught
+    by generic `except Exception` handlers.
+    """
+    
+    _name = "ask_user"
+    _description = "Ask the user for clarification or additional information. Use this when you need more details to complete a task."
+    _parameters = {
+        "type": "object",
+        "properties": {
+            "question": {
+                "type": "string",
+                "description": "The question to ask the user",
+            },
+            "options": {
+                "type": "array",
+                "items": {"type": "string"},
+                "description": "Optional list of suggested answers",
+            },
+            "context": {
+                "type": "string",
+                "description": "Additional context about why you're asking",
+            },
+        },
+        "required": ["question"],
+    }
+    
+    async def execute(
+        self,
+        question: str,
+        options: list[str] | None = None,
+        context: str | None = None,
+        *,
+        ctx: "InvocationContext | None" = None,
+    ) -> ToolResult:
+        """Execute ask_user tool.
+        
+        This method does NOT return normally - it raises HITLSuspend
+        to pause the agent execution.
+        
+        Args:
+            question: Question to ask the user
+            options: Optional predefined answer options
+            context: Additional context
+            ctx: Invocation context (injected by agent)
+            
+        Returns:
+            Never returns normally
+            
+        Raises:
+            HITLSuspend: Always raised to suspend execution
+        """
+        if ctx is None:
+            # If no context, return error (should not happen in normal use)
+            return ToolResult.error("Cannot ask user without execution context")
+        
+        # Generate request ID
+        request_id = generate_id("req")
+        
+        # Create HITL request data
+        request = HITLRequest(
+            request_id=request_id,
+            request_type="ask_user",
+            message=question,
+            options=options,
+            tool_name=self._name,
+            metadata={"context": context} if context else {},
+        )
+        
+        # Checkpoint current state
+        if hasattr(ctx, "state") and ctx.state is not None:
+            await ctx.state.checkpoint()
+        
+        # Update invocation status to SUSPENDED
+        if ctx.backends and ctx.backends.invocation:
+            await ctx.backends.invocation.update(ctx.invocation_id, {
+                "status": "suspended",
+                "pending_request_id": request_id,
+                "pending_request_type": "ask_user",
+                "pending_request_data": request.to_dict(),
+            })
+        
+        # Emit HITL request block to frontend
+        await ctx.emit(BlockEvent(
+            kind="hitl_request",
+            data={
+                "request_id": request_id,
+                "type": "ask_user",
+                "question": question,
+                "options": options,
+                "context": context,
+            },
+        ))
+        
+        # Raise signal to suspend execution
+        raise HITLSuspend(
+            request_id=request_id,
+            request_type="ask_user",
+            message=question,
+            options=options,
+            tool_name=self._name,
+            metadata={"context": context} if context else {},
+        )
+
+
+class ConfirmTool(BaseTool):
+    """Tool for LLM to request confirmation before proceeding.
+    
+    Similar to ask_user but specifically for yes/no confirmations.
+    """
+    
+    _name = "confirm"
+    _description = "Ask the user to confirm an action before proceeding."
+    _parameters = {
+        "type": "object",
+        "properties": {
+            "action": {
+                "type": "string",
+                "description": "The action you want to perform",
+            },
+            "details": {
+                "type": "string",
+                "description": "Details about the action",
+            },
+            "risk_level": {
+                "type": "string",
+                "enum": ["low", "medium", "high"],
+                "description": "Risk level of the action",
+            },
+        },
+        "required": ["action"],
+    }
+    
+    async def execute(
+        self,
+        action: str,
+        details: str | None = None,
+        risk_level: str = "medium",
+        *,
+        ctx: "InvocationContext | None" = None,
+    ) -> ToolResult:
+        """Execute confirm tool.
+        
+        Args:
+            action: Action to confirm
+            details: Additional details
+            risk_level: Risk level (low, medium, high)
+            ctx: Invocation context
+            
+        Raises:
+            HITLSuspend: Always raised to suspend execution
+        """
+        if ctx is None:
+            return ToolResult.error("Cannot confirm without execution context")
+        
+        request_id = generate_id("req")
+        
+        message = f"Confirm: {action}"
+        if details:
+            message += f"\n\nDetails: {details}"
+        
+        request = HITLRequest(
+            request_id=request_id,
+            request_type="confirm",
+            message=message,
+            options=["Yes, proceed", "No, cancel"],
+            tool_name=self._name,
+            metadata={
+                "action": action,
+                "details": details,
+                "risk_level": risk_level,
+            },
+        )
+        
+        # Checkpoint
+        if hasattr(ctx, "state") and ctx.state is not None:
+            await ctx.state.checkpoint()
+        
+        # Update invocation
+        if ctx.backends and ctx.backends.invocation:
+            await ctx.backends.invocation.update(ctx.invocation_id, {
+                "status": "suspended",
+                "pending_request_id": request_id,
+                "pending_request_type": "confirm",
+                "pending_request_data": request.to_dict(),
+            })
+        
+        # Emit block
+        await ctx.emit(BlockEvent(
+            kind="hitl_request",
+            data={
+                "request_id": request_id,
+                "type": "confirm",
+                "action": action,
+                "details": details,
+                "risk_level": risk_level,
+                "options": ["Yes, proceed", "No, cancel"],
+            },
+        ))
+        
+        raise HITLSuspend(
+            request_id=request_id,
+            request_type="confirm",
+            message=message,
+            options=["Yes, proceed", "No, cancel"],
+            tool_name=self._name,
+            metadata=request.metadata,
+        )
+
+
+__all__ = ["AskUserTool", "ConfirmTool"]