RouteKitAI 0.1.0__py3-none-any.whl

routekitai/core/tool.py

@@ -0,0 +1,148 @@
+"""Tool primitive for RouteKit."""
+
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Any, TypeVar
+
+from pydantic import BaseModel, Field, create_model
+
+from routekitai.core.errors import ToolError
+
+TInput = TypeVar("TInput", bound=BaseModel)
+TOutput = TypeVar("TOutput", bound=BaseModel)
+
+
+class ToolPermission(str, Enum):
+    """Tool permission types."""
+
+    NETWORK = "network"
+    FILESYSTEM = "filesystem"
+    DATABASE = "database"
+    NONE = "none"
+
+
+class Tool(BaseModel, ABC):
+    """Base class for tools with pydantic input/output models."""
+
+    model_config = {"arbitrary_types_allowed": True}
+
+    name: str = Field(..., description="Tool name")
+    description: str = Field(..., description="Tool description")
+    input_model: type[BaseModel] | None = Field(
+        default=None, description="Pydantic model for input validation"
+    )
+    output_model: type[BaseModel] | None = Field(
+        default=None, description="Pydantic model for output validation"
+    )
+    permissions: list[ToolPermission] = Field(
+        default_factory=list, description="Required permissions"
+    )
+    rate_limit: int | None = Field(default=None, description="Rate limit (calls per second)")
+    timeout: float | None = Field(default=None, description="Timeout in seconds")
+    redact_fields: list[str] = Field(
+        default_factory=list,
+        description="Field names to redact in traces (e.g., ['api_key', 'password'])",
+    )
+
+    @property
+    def parameters(self) -> dict[str, Any]:
+        """Generate JSON Schema from input_model.
+
+        Returns:
+            JSON Schema dictionary
+        """
+        if self.input_model is None:
+            return {"type": "object", "properties": {}}
+        return self.input_model.model_json_schema()
+
+    def redact_data(self, data: dict[str, Any]) -> dict[str, Any]:
+        """Redact sensitive fields from data (handles nested dicts).
+
+        Args:
+            data: Data dictionary to redact
+
+        Returns:
+            Data with redacted fields
+        """
+        if not self.redact_fields:
+            return data
+
+        def _redact_recursive(obj: Any) -> Any:
+            """Recursively redact fields in nested structures."""
+            if isinstance(obj, dict):
+                redacted: dict[str, Any] = {}
+                for key, value in obj.items():
+                    if key in self.redact_fields:
+                        redacted[key] = "[REDACTED]"
+                    elif isinstance(value, (dict, list)):
+                        redacted[key] = _redact_recursive(value)
+                    else:
+                        redacted[key] = value
+                return redacted
+            elif isinstance(obj, list):
+                return [_redact_recursive(item) for item in obj]
+            else:
+                return obj
+
+        result = _redact_recursive(data)
+        assert isinstance(result, dict)
+        return result
+
+    @abstractmethod
+    async def run(self, input: BaseModel) -> BaseModel:
+        """Execute the tool with validated input.
+
+        Args:
+            input: Validated input model instance
+
+        Returns:
+            Validated output model instance
+
+        Raises:
+            ToolError: If tool execution fails
+        """
+        raise NotImplementedError("Subclasses must implement run")
+
+    async def execute(self, **kwargs: Any) -> Any:
+        """Execute tool with raw kwargs (validates input/output).
+
+        Args:
+            **kwargs: Raw tool arguments
+
+        Returns:
+            Tool output (validated if output_model is set)
+
+        Raises:
+            ToolError: If validation or execution fails
+        """
+        try:
+            # Validate input
+            if self.input_model is not None:
+                input_instance = self.input_model(**kwargs)
+            else:
+                # Create a minimal model if no input_model
+                if not kwargs:
+                    # Empty kwargs - create empty model
+                    InputModel = create_model("InputModel")
+                    input_instance = InputModel()
+                else:
+                    # Dynamically create model from kwargs
+                    field_definitions: dict[str, Any] = {}
+                    for k, v in kwargs.items():
+                        field_definitions[k] = (type(v), ...)
+                    InputModel = create_model("InputModel", **field_definitions)
+                    input_instance = InputModel(**kwargs)
+
+            # Execute
+            output = await self.run(input_instance)
+
+            # Validate output
+            if self.output_model is not None and not isinstance(output, self.output_model):
+                raise ToolError(f"Tool {self.name} returned invalid output type")
+
+            return output
+
+        except Exception as e:
+            if isinstance(e, ToolError):
+                raise
+            raise ToolError(f"Tool {self.name} execution failed: {e}") from e

routekitai/core/tools.py

@@ -0,0 +1,180 @@
+"""Built-in tools for RouteKit."""
+
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from routekitai.core.errors import ToolError
+from routekitai.core.tool import Tool, ToolPermission
+
+
+class EchoInput(BaseModel):
+    """Input for EchoTool."""
+
+    message: str = Field(..., description="Message to echo")
+
+
+class EchoOutput(BaseModel):
+    """Output for EchoTool."""
+
+    echoed: str = Field(..., description="Echoed message")
+
+
+class EchoTool(Tool):
+    """Echo tool for testing.
+
+    Simply echoes back the input message. Useful for testing and debugging.
+    """
+
+    model_config = {"arbitrary_types_allowed": True}
+
+    def __init__(self) -> None:
+        super().__init__(
+            name="echo",
+            description="Echo back a message (useful for testing)",
+            input_model=EchoInput,
+            output_model=EchoOutput,
+        )
+
+    async def run(self, input: BaseModel) -> BaseModel:
+        """Echo the input message.
+
+        Args:
+            input: EchoInput instance
+
+        Returns:
+            EchoOutput instance
+        """
+        if not isinstance(input, EchoInput):
+            raise ToolError("Invalid input type for EchoTool")
+        return EchoOutput(echoed=input.message)
+
+
+class HttpGetInput(BaseModel):
+    """Input for HttpGetTool."""
+
+    url: str = Field(..., description="URL to fetch")
+    headers: dict[str, str] = Field(default_factory=dict, description="HTTP headers")
+    timeout: float = Field(default=30.0, description="Request timeout in seconds")
+
+
+class HttpGetOutput(BaseModel):
+    """Output for HttpGetTool."""
+
+    status_code: int = Field(..., description="HTTP status code")
+    headers: dict[str, str] = Field(..., description="Response headers")
+    body: str = Field(..., description="Response body")
+
+
+class HttpGetTool(Tool):
+    """HTTP GET tool.
+
+    Requires NETWORK permission. Redacts 'api_key' and 'authorization' from headers.
+    """
+
+    model_config = {"arbitrary_types_allowed": True}
+
+    def __init__(self) -> None:
+        super().__init__(
+            name="http_get",
+            description="Perform HTTP GET request",
+            input_model=HttpGetInput,
+            output_model=HttpGetOutput,
+            permissions=[ToolPermission.NETWORK],
+            redact_fields=["api_key", "authorization"],
+        )
+
+    async def run(self, input: BaseModel) -> BaseModel:
+        """Execute HTTP GET request.
+
+        Args:
+            input: HttpGetInput instance
+
+        Returns:
+            HttpGetOutput instance
+
+        Raises:
+            ToolError: If request fails
+        """
+        if not isinstance(input, HttpGetInput):
+            raise ToolError("Invalid input type for HttpGetTool")
+
+        try:
+            # Try to import httpx (optional dependency)
+            try:
+                import httpx
+            except ImportError:
+                raise ToolError(
+                    "httpx is required for HttpGetTool. Install with: pip install httpx"
+                ) from None
+
+            async with httpx.AsyncClient(timeout=input.timeout) as client:
+                response = await client.get(input.url, headers=input.headers)
+                return HttpGetOutput(
+                    status_code=response.status_code,
+                    headers=dict(response.headers),
+                    body=response.text,
+                )
+        except Exception as e:
+            raise ToolError(f"HTTP GET failed: {e}") from e
+
+
+class FileReadInput(BaseModel):
+    """Input for FileReadTool."""
+
+    path: str = Field(..., description="File path to read")
+    encoding: str = Field(default="utf-8", description="File encoding")
+
+
+class FileReadOutput(BaseModel):
+    """Output for FileReadTool."""
+
+    content: str = Field(..., description="File content")
+    size: int = Field(..., description="File size in bytes")
+
+
+class FileReadTool(Tool):
+    """File read tool.
+
+    Requires FILESYSTEM permission. Reads file content.
+    """
+
+    model_config = {"arbitrary_types_allowed": True}
+
+    def __init__(self) -> None:
+        super().__init__(
+            name="file_read",
+            description="Read content from a file",
+            input_model=FileReadInput,
+            output_model=FileReadOutput,
+            permissions=[ToolPermission.FILESYSTEM],
+        )
+
+    async def run(self, input: BaseModel) -> BaseModel:
+        """Read file content.
+
+        Args:
+            input: FileReadInput instance
+
+        Returns:
+            FileReadOutput instance
+
+        Raises:
+            ToolError: If file read fails
+        """
+        if not isinstance(input, FileReadInput):
+            raise ToolError("Invalid input type for FileReadTool")
+
+        try:
+            file_path = Path(input.path)
+            if not file_path.exists():
+                raise ToolError(f"File not found: {input.path}")
+            if not file_path.is_file():
+                raise ToolError(f"Path is not a file: {input.path}")
+
+            content = file_path.read_text(encoding=input.encoding)
+            size = file_path.stat().st_size
+
+            return FileReadOutput(content=content, size=size)
+        except Exception as e:
+            raise ToolError(f"File read failed: {e}") from e

routekitai/evals/__init__.py

@@ -0,0 +1,13 @@
+"""Evaluation harness for routkitai agents."""
+
+from routekitai.evals.dataset import Dataset
+from routekitai.evals.metrics import ContainsMetric, ExactMatchMetric, RegexMetric
+from routekitai.evals.runner import EvalRunner
+
+__all__ = [
+    "Dataset",
+    "EvalRunner",
+    "ExactMatchMetric",
+    "ContainsMetric",
+    "RegexMetric",
+]

routekitai/evals/dataset.py

@@ -0,0 +1,75 @@
+"""Dataset format for evaluations."""
+
+import json
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class EvalExample(BaseModel):
+    """A single evaluation example."""
+
+    id: str = Field(..., description="Example ID")
+    input: str = Field(..., description="Input prompt")
+    expected_output: str | None = Field(default=None, description="Expected output")
+    metadata: dict[str, Any] = Field(default_factory=dict, description="Additional metadata")
+
+
+class Dataset(BaseModel):
+    """Evaluation dataset."""
+
+    name: str = Field(..., description="Dataset name")
+    examples: list[EvalExample] = Field(default_factory=list, description="Evaluation examples")
+
+    @classmethod
+    def from_jsonl(cls, file_path: Path | str) -> "Dataset":
+        """Load dataset from JSONL file.
+
+        Args:
+            file_path: Path to JSONL file
+
+        Returns:
+            Dataset instance
+        """
+        file_path = Path(file_path)
+        examples = []
+
+        with open(file_path, encoding="utf-8") as f:
+            for line_num, line in enumerate(f, 1):
+                line = line.strip()
+                if not line:
+                    continue
+
+                try:
+                    data = json.loads(line)
+                    example = EvalExample(
+                        id=data.get("id", f"example_{line_num}"),
+                        input=data.get("input", data.get("prompt", "")),
+                        expected_output=data.get("expected_output", data.get("output")),
+                        metadata=data.get("metadata", {}),
+                    )
+                    examples.append(example)
+                except json.JSONDecodeError as e:
+                    raise ValueError(f"Invalid JSON on line {line_num}: {e}") from e
+
+        return cls(name=file_path.stem, examples=examples)
+
+    def to_jsonl(self, file_path: Path | str) -> None:
+        """Save dataset to JSONL file.
+
+        Args:
+            file_path: Path to save JSONL file
+        """
+        file_path = Path(file_path)
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+
+        with open(file_path, "w", encoding="utf-8") as f:
+            for example in self.examples:
+                data = {
+                    "id": example.id,
+                    "input": example.input,
+                    "expected_output": example.expected_output,
+                    "metadata": example.metadata,
+                }
+                f.write(json.dumps(data) + "\n")

routekitai/evals/metrics.py

@@ -0,0 +1,101 @@
+"""Evaluation metrics for agent outputs."""
+
+import re
+from abc import ABC, abstractmethod
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class Metric(BaseModel, ABC):
+    """Base class for evaluation metrics."""
+
+    name: str = Field(..., description="Metric name")
+
+    @abstractmethod
+    def score(self, expected: str | None, actual: str) -> float:
+        """Score an output against expected output.
+
+        Args:
+            expected: Expected output (can be None)
+            actual: Actual output
+
+        Returns:
+            Score between 0.0 and 1.0
+        """
+        raise NotImplementedError("Subclasses must implement score")
+
+
+class ExactMatchMetric(Metric):
+    """Exact match metric (case-insensitive)."""
+
+    name: str = Field(default="exact_match", description="Metric name")
+    case_sensitive: bool = Field(default=False, description="Whether to be case-sensitive")
+
+    def score(self, expected: str | None, actual: str) -> float:
+        """Score based on exact match.
+
+        Args:
+            expected: Expected output
+            actual: Actual output
+
+        Returns:
+            1.0 if exact match, 0.0 otherwise
+        """
+        if expected is None:
+            return 0.0
+
+        if self.case_sensitive:
+            return 1.0 if expected.strip() == actual.strip() else 0.0
+        else:
+            return 1.0 if expected.strip().lower() == actual.strip().lower() else 0.0
+
+
+class ContainsMetric(Metric):
+    """Contains metric (checks if expected is contained in actual)."""
+
+    name: str = Field(default="contains", description="Metric name")
+    case_sensitive: bool = Field(default=False, description="Whether to be case-sensitive")
+
+    def score(self, expected: str | None, actual: str) -> float:
+        """Score based on substring match.
+
+        Args:
+            expected: Expected output (substring to find)
+            actual: Actual output
+
+        Returns:
+            1.0 if expected is contained in actual, 0.0 otherwise
+        """
+        if expected is None:
+            return 0.0
+
+        if self.case_sensitive:
+            return 1.0 if expected.strip() in actual else 0.0
+        else:
+            return 1.0 if expected.strip().lower() in actual.lower() else 0.0
+
+
+class RegexMetric(Metric):
+    """Regex pattern matching metric."""
+
+    name: str = Field(default="regex", description="Metric name")
+    pattern: str = Field(..., description="Regex pattern to match")
+    flags: int = Field(default=0, description="Regex flags")
+
+    def __init__(self, **data: Any) -> None:
+        """Initialize regex metric."""
+        super().__init__(**data)
+        self._compiled_pattern = re.compile(self.pattern, self.flags)
+
+    def score(self, expected: str | None, actual: str) -> float:
+        """Score based on regex match.
+
+        Args:
+            expected: Expected output (not used, pattern is used instead)
+            actual: Actual output
+
+        Returns:
+            1.0 if pattern matches, 0.0 otherwise
+        """
+        return 1.0 if self._compiled_pattern.search(actual) else 0.0

routekitai/evals/runner.py

@@ -0,0 +1,184 @@
+"""Evaluation runner for agent testing."""
+
+import json
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from routekitai.core.runtime import Runtime
+from routekitai.evals.dataset import Dataset
+from routekitai.evals.metrics import Metric
+
+
+class EvalResult(BaseModel):
+    """Result for a single evaluation example."""
+
+    example_id: str = Field(..., description="Example ID")
+    input: str = Field(..., description="Input prompt")
+    expected_output: str | None = Field(default=None, description="Expected output")
+    actual_output: str = Field(..., description="Actual output")
+    scores: dict[str, float] = Field(default_factory=dict, description="Metric scores")
+    trace_id: str | None = Field(default=None, description="Trace ID")
+    passed: bool = Field(default=False, description="Whether evaluation passed")
+    error: str | None = Field(default=None, description="Error if evaluation failed")
+
+
+class EvalReport(BaseModel):
+    """Evaluation report with aggregated results."""
+
+    dataset_name: str = Field(..., description="Dataset name")
+    agent_name: str = Field(..., description="Agent name")
+    total_examples: int = Field(..., description="Total number of examples")
+    passed: int = Field(..., description="Number of passed examples")
+    failed: int = Field(..., description="Number of failed examples")
+    errors: int = Field(..., description="Number of errors")
+    average_scores: dict[str, float] = Field(
+        default_factory=dict, description="Average scores per metric"
+    )
+    results: list[EvalResult] = Field(default_factory=list, description="Individual results")
+
+
+class EvalRunner(BaseModel):
+    """Runner for evaluating agents on datasets."""
+
+    runtime: Runtime = Field(..., description="Runtime for agent execution")
+    metrics: list[Metric] = Field(default_factory=list, description="Metrics to compute")
+    regression_mode: bool = Field(
+        default=False, description="Enable regression mode (compare to baseline)"
+    )
+    baseline_dir: Path | None = Field(default=None, description="Directory with baseline traces")
+
+    async def run(self, agent_name: str, dataset: Dataset, **kwargs: Any) -> EvalReport:
+        """Run evaluation on a dataset.
+
+        Args:
+            agent_name: Name of agent to evaluate
+            dataset: Evaluation dataset
+            **kwargs: Additional parameters for agent execution
+
+        Returns:
+            EvalReport with results
+        """
+        if agent_name not in self.runtime.agents:
+            raise ValueError(f"Agent '{agent_name}' not found in runtime")
+
+        results: list[EvalResult] = []
+        passed = 0
+        failed = 0
+        errors = 0
+
+        for example in dataset.examples:
+            try:
+                # Execute agent
+                result = await self.runtime.run(agent_name, example.input, **kwargs)
+                actual_output = result.output.content
+
+                # Compute scores
+                scores: dict[str, float] = {}
+                for metric in self.metrics:
+                    score = metric.score(example.expected_output, actual_output)
+                    scores[metric.name] = score
+
+                # Check if passed (at least one metric must score > 0)
+                passed_example = any(score > 0 for score in scores.values()) if scores else False
+
+                # Regression mode: compare to baseline
+                if self.regression_mode and self.baseline_dir:
+                    baseline_result = await self._load_baseline_result(example.id)
+                    if baseline_result:
+                        if baseline_result.actual_output != actual_output:
+                            # Output changed - mark as regression
+                            passed_example = False
+                            scores["regression"] = 0.0
+                        else:
+                            scores["regression"] = 1.0
+
+                if passed_example:
+                    passed += 1
+                else:
+                    failed += 1
+
+                eval_result = EvalResult(
+                    example_id=example.id,
+                    input=example.input,
+                    expected_output=example.expected_output,
+                    actual_output=actual_output,
+                    scores=scores,
+                    trace_id=result.trace_id,
+                    passed=passed_example,
+                )
+                results.append(eval_result)
+
+            except Exception as e:
+                errors += 1
+                eval_result = EvalResult(
+                    example_id=example.id,
+                    input=example.input,
+                    expected_output=example.expected_output,
+                    actual_output="",
+                    error=str(e),
+                    passed=False,
+                )
+                results.append(eval_result)
+
+        # Compute average scores
+        average_scores: dict[str, float] = {}
+        if results:
+            metric_names: set[str] = set()
+            for eval_result in results:
+                metric_names.update(eval_result.scores.keys())
+
+            for metric_name in metric_names:
+                scores_list = [r.scores.get(metric_name, 0.0) for r in results if r.error is None]
+                if scores_list:
+                    average_scores[metric_name] = sum(scores_list) / len(scores_list)
+
+        return EvalReport(
+            dataset_name=dataset.name,
+            agent_name=agent_name,
+            total_examples=len(dataset.examples),
+            passed=passed,
+            failed=failed,
+            errors=errors,
+            average_scores=average_scores,
+            results=results,
+        )
+
+    async def _load_baseline_result(self, example_id: str) -> EvalResult | None:
+        """Load baseline result for regression comparison.
+
+        Args:
+            example_id: Example ID
+
+        Returns:
+            Baseline result or None
+        """
+        if not self.baseline_dir:
+            return None
+
+        baseline_file = self.baseline_dir / f"{example_id}.json"
+        if not baseline_file.exists():
+            return None
+
+        try:
+            with open(baseline_file, encoding="utf-8") as f:
+                data = json.load(f)
+                return EvalResult(**data)
+        except Exception:
+            return None
+
+    def save_baseline(self, report: EvalReport, output_dir: Path) -> None:
+        """Save evaluation results as baseline for regression testing.
+
+        Args:
+            report: Evaluation report
+            output_dir: Directory to save baseline results
+        """
+        output_dir.mkdir(parents=True, exist_ok=True)
+
+        for result in report.results:
+            if result.error is None:
+                baseline_file = output_dir / f"{result.example_id}.json"
+                with open(baseline_file, "w", encoding="utf-8") as f:
+                    json.dump(result.model_dump(), f, indent=2)