prompteval-core 0.1.0__py3-none-any.whl

prompteval-core/client.py

@@ -0,0 +1,443 @@
+"""
+PromptEval SDK Client
+
+A Python client for the PromptEval API - Semantic Testing for LLMs.
+
+Usage:
+    from prompteval import PromptEval
+    
+    client = PromptEval(api_key="pe_xxxxx")
+    result = client.run_from_files("adapter.yml", "tests.yml")
+    
+    if result.success:
+        print("All tests passed!")
+    else:
+        for test in result.failed_tests:
+            print(f"Failed: {test.id_test}")
+"""
+
+import requests
+import yaml
+from pathlib import Path
+from dataclasses import dataclass, field
+from typing import Optional, List, Dict, Any
+
+from .exceptions import (
+    AuthenticationError,
+    QuotaExceededError,
+    RateLimitError,
+    ValidationError,
+    APIError,
+    ConnectionError,
+    TimeoutError,
+)
+
+
+# =============================================================================
+# DATA CLASSES
+# =============================================================================
+
+@dataclass
+class TestResult:
+    """Individual test result"""
+    id_test: str
+    description: str
+    passed: bool
+    duration_ms: float
+    validation_method: str
+    expected: Optional[str] = None
+    actual: Optional[str] = None
+    similarity: Optional[float] = None
+    threshold: Optional[float] = None
+    error: Optional[str] = None
+    custom_fields: Dict[str, Any] = field(default_factory=dict)
+    
+    @property
+    def failed(self) -> bool:
+        return not self.passed and not self.error
+    
+    def __repr__(self) -> str:
+        status = "✅" if self.passed else "❌"
+        return f"{status} [{self.id_test}] {self.description}"
+
+
+@dataclass
+class EvalResult:
+    """Evaluation result containing all test results"""
+    success_rate: float
+    total_tests: int
+    total_passed: int
+    total_failed: int
+    total_errors: int
+    duration_ms: float
+    model_used: str
+    test_results: List[TestResult]
+    config_used: Optional[Dict[str, Any]] = None
+    
+    @property
+    def success(self) -> bool:
+        """Returns True if all tests passed"""
+        return self.total_failed == 0 and self.total_errors == 0
+    
+    @property
+    def failed_tests(self) -> List[TestResult]:
+        """Returns list of failed tests"""
+        return [t for t in self.test_results if not t.passed]
+    
+    @property
+    def passed_tests(self) -> List[TestResult]:
+        """Returns list of passed tests"""
+        return [t for t in self.test_results if t.passed]
+    
+    @property
+    def error_tests(self) -> List[TestResult]:
+        """Returns list of tests with errors"""
+        return [t for t in self.test_results if t.error]
+    
+    def __repr__(self) -> str:
+        status = "✅ PASSED" if self.success else "❌ FAILED"
+        return (
+            f"EvalResult({status}: {self.total_passed}/{self.total_tests} tests, "
+            f"{self.success_rate}% success rate, {self.duration_ms:.0f}ms)"
+        )
+
+
+@dataclass
+class License:
+    """User license information"""
+    id: str
+    plan: str
+    is_active: bool
+    max_tests_per_month: int
+    tests_used_this_month: int = 0
+    tests_remaining: int = 0
+    created_at: Optional[str] = None
+    expires_at: Optional[str] = None
+
+
+@dataclass
+class User:
+    """User information"""
+    id: str
+    email: str
+    full_name: Optional[str]
+    role: str
+    licenses: List[License] = field(default_factory=list)
+
+
+# =============================================================================
+# MAIN CLIENT
+# =============================================================================
+
+class PromptEval:
+    """
+    PromptEval API Client
+    
+    Args:
+        api_key: Your PromptEval API key (starts with 'pe_')
+        base_url: API base URL (default: https://api.getprompteval.com)
+        timeout: Request timeout in seconds (default: 300)
+    
+    Example:
+        >>> client = PromptEval(api_key="pe_xxxxx")
+        >>> result = client.run_from_files("adapter.yml", "tests.yml")
+        >>> print(result.success_rate)
+    """
+    
+    DEFAULT_URL = "https://api.getprompteval.com"
+    DEFAULT_TIMEOUT = 300
+    
+    def __init__(
+        self,
+        api_key: str,
+        base_url: str = None,
+        timeout: int = None
+    ):
+        if not api_key:
+            raise ValidationError("API key is required")
+        
+        self.api_key = api_key
+        self.base_url = (base_url or self.DEFAULT_URL).rstrip("/")
+        self.timeout = timeout or self.DEFAULT_TIMEOUT
+        
+        self._session = requests.Session()
+        self._session.headers.update({
+            "Content-Type": "application/json",
+            "X-API-Key": api_key,
+        })
+    
+    # =========================================================================
+    # EVALUATION METHODS
+    # =========================================================================
+    
+    def run(
+        self,
+        adapter: Dict[str, Any],
+        tests: List[Dict[str, Any]],
+        model: str = "gpt-4o",
+        config: Dict[str, Any] = None
+    ) -> EvalResult:
+        """
+        Run evaluation with adapter and tests as dictionaries.
+        
+        Args:
+            adapter: Adapter configuration dict
+            tests: List of test case dicts
+            model: Model identifier (default: gpt-4o)
+            config: Optional runner configuration
+            
+        Returns:
+            EvalResult with all test results
+        """
+        # Combine adapter + tests
+        adapter_copy = adapter.copy()
+        adapter_copy["tests"] = tests
+        adapter_yaml = yaml.dump(adapter_copy, allow_unicode=True)
+        
+        return self._execute_eval(adapter_yaml, model, config)
+    
+    def run_from_files(
+        self,
+        adapter_path: str,
+        tests_path: str = None,
+        model: str = "gpt-4o",
+        config: Dict[str, Any] = None
+    ) -> EvalResult:
+        """
+        Run evaluation from YAML files.
+        
+        Args:
+            adapter_path: Path to adapter YAML file
+            tests_path: Path to tests YAML file (optional if embedded in adapter)
+            model: Model identifier (default: gpt-4o)
+            config: Optional runner configuration
+            
+        Returns:
+            EvalResult with all test results
+        """
+        # Read adapter
+        adapter_file = Path(adapter_path)
+        if not adapter_file.exists():
+            raise ValidationError(f"Adapter file not found: {adapter_path}")
+        
+        with open(adapter_file, "r", encoding="utf-8") as f:
+            adapter_yaml = f.read()
+            adapter_data = yaml.safe_load(adapter_yaml)
+        
+        # Read tests if separate file
+        if tests_path:
+            tests_file = Path(tests_path)
+            if not tests_file.exists():
+                raise ValidationError(f"Tests file not found: {tests_path}")
+            
+            with open(tests_file, "r", encoding="utf-8") as f:
+                tests_data = yaml.safe_load(f)
+            
+            adapter_data["tests"] = tests_data.get("tests", tests_data)
+            adapter_yaml = yaml.dump(adapter_data, allow_unicode=True)
+        
+        return self._execute_eval(adapter_yaml, model, config)
+    
+    def run_from_yaml(
+        self,
+        adapter_yaml: str,
+        model: str = "gpt-4o",
+        config: Dict[str, Any] = None
+    ) -> EvalResult:
+        """
+        Run evaluation from raw YAML string.
+        
+        Args:
+            adapter_yaml: Complete adapter YAML with embedded tests
+            model: Model identifier (default: gpt-4o)
+            config: Optional runner configuration
+            
+        Returns:
+            EvalResult with all test results
+        """
+        return self._execute_eval(adapter_yaml, model, config)
+    
+    def _execute_eval(
+        self,
+        adapter_yaml: str,
+        model: str,
+        config: Dict[str, Any] = None
+    ) -> EvalResult:
+        """Execute evaluation against the API"""
+        
+        payload = {
+            "adapter_yaml": adapter_yaml,
+            "model": model,
+        }
+        if config:
+            payload["config"] = config
+        
+        data = self._request("POST", "/v1/evals/", json=payload)
+        
+        return EvalResult(
+            success_rate=data["success_rate"],
+            total_tests=data["total_tests"],
+            total_passed=data["total_passed"],
+            total_failed=data["total_failed"],
+            total_errors=data["total_errors"],
+            duration_ms=data["duration_ms"],
+            model_used=data["model_used"],
+            config_used=data.get("config_used"),
+            test_results=[
+                TestResult(
+                    id_test=t["id_test"],
+                    description=t["description"],
+                    passed=t["passed"],
+                    duration_ms=t["duration_ms"],
+                    validation_method=t["validation_method"],
+                    expected=t.get("expected"),
+                    actual=t.get("actual"),
+                    similarity=t.get("similarity"),
+                    threshold=t.get("threshold"),
+                    error=t.get("error"),
+                    custom_fields=t.get("custom_fields", {}),
+                )
+                for t in data["test_results"]
+            ],
+        )
+    
+    # =========================================================================
+    # ACCOUNT METHODS
+    # =========================================================================
+    
+    def get_licenses(self) -> List[License]:
+        """Get all licenses for the current user"""
+        data = self._request("GET", "/v1/client/licenses")
+        
+        return [
+            License(
+                id=lic["id"],
+                plan=lic["plan"],
+                is_active=lic["is_active"],
+                max_tests_per_month=lic["max_tests_per_month"],
+                tests_used_this_month=lic.get("tests_used_this_month", 0),
+                tests_remaining=lic.get("tests_remaining", 0),
+                created_at=lic.get("created_at"),
+                expires_at=lic.get("expires_at"),
+            )
+            for lic in data["licenses"]
+        ]
+    
+    def get_usage(self, license_id: str) -> Dict[str, Any]:
+        """Get usage statistics for a license"""
+        return self._request("GET", f"/v1/client/licenses/{license_id}/usage")
+    
+    def get_api_keys(self, license_id: str) -> List[Dict[str, Any]]:
+        """Get API keys for a license"""
+        data = self._request("GET", f"/v1/client/licenses/{license_id}/api-keys")
+        return data["api_keys"]
+    
+    # =========================================================================
+    # UTILITY METHODS
+    # =========================================================================
+    
+    def health(self) -> Dict[str, Any]:
+        """Check API health status"""
+        return self._request("GET", "/health")
+    
+    def validate_yaml(self, file_path: str) -> bool:
+        """
+        Validate a YAML file locally.
+        
+        Args:
+            file_path: Path to YAML file
+            
+        Returns:
+            True if valid, False otherwise
+        """
+        try:
+            with open(file_path, "r", encoding="utf-8") as f:
+                data = yaml.safe_load(f)
+            return "tests" in data or isinstance(data, list)
+        except Exception:
+            return False
+    
+    # =========================================================================
+    # INTERNAL METHODS
+    # =========================================================================
+    
+    def _request(
+        self,
+        method: str,
+        endpoint: str,
+        **kwargs
+    ) -> Dict[str, Any]:
+        """Make HTTP request to API"""
+        
+        url = f"{self.base_url}{endpoint}"
+        
+        try:
+            response = self._session.request(
+                method=method,
+                url=url,
+                timeout=self.timeout,
+                **kwargs
+            )
+        except requests.exceptions.Timeout:
+            raise TimeoutError(f"Request timed out after {self.timeout}s")
+        except requests.exceptions.ConnectionError as e:
+            raise ConnectionError(f"Cannot connect to API: {self.base_url}") from e
+        
+        # Handle errors
+        if response.status_code == 401:
+            raise AuthenticationError("Invalid API key")
+        elif response.status_code == 402:
+            raise QuotaExceededError("Monthly quota exceeded")
+        elif response.status_code == 403:
+            detail = response.json().get("detail", "Access denied")
+            raise AuthenticationError(detail)
+        elif response.status_code == 429:
+            raise RateLimitError("Rate limit exceeded")
+        elif response.status_code >= 400:
+            try:
+                error_data = response.json()
+                message = error_data.get("detail") or error_data.get("error", "Unknown error")
+            except Exception:
+                message = response.text or f"HTTP {response.status_code}"
+            raise APIError(message, status_code=response.status_code)
+        
+        return response.json()
+    
+    def __repr__(self) -> str:
+        return f"PromptEval(base_url='{self.base_url}')"
+
+
+# =============================================================================
+# CONVENIENCE FUNCTIONS
+# =============================================================================
+
+def run_tests(
+    adapter_path: str,
+    tests_path: str = None,
+    api_key: str = None,
+    base_url: str = None,
+    model: str = "gpt-4o"
+) -> EvalResult:
+    """
+    Convenience function to run tests without instantiating client.
+    
+    Args:
+        adapter_path: Path to adapter YAML file
+        tests_path: Path to tests YAML file (optional)
+        api_key: API key (or set PROMPTEVAL_API_KEY env var)
+        base_url: API URL (optional)
+        model: Model identifier
+        
+    Returns:
+        EvalResult
+    """
+    import os
+    
+    key = api_key or os.environ.get("PROMPTEVAL_API_KEY")
+    if not key:
+        raise ValidationError(
+            "API key required. Pass api_key or set PROMPTEVAL_API_KEY environment variable."
+        )
+    
+    client = PromptEval(api_key=key, base_url=base_url)
+    return client.run_from_files(adapter_path, tests_path, model=model)

prompteval-core/exceptions.py

@@ -0,0 +1,47 @@
+"""
+PromptEval SDK Exceptions
+"""
+
+
+class PromptEvalError(Exception):
+    """Base exception for PromptEval SDK"""
+    pass
+
+
+class AuthenticationError(PromptEvalError):
+    """Invalid or missing API key"""
+    pass
+
+
+class QuotaExceededError(PromptEvalError):
+    """Monthly quota exceeded"""
+    pass
+
+
+class RateLimitError(PromptEvalError):
+    """Rate limit exceeded"""
+    pass
+
+
+class ValidationError(PromptEvalError):
+    """Invalid input data"""
+    pass
+
+
+class APIError(PromptEvalError):
+    """Generic API error"""
+    
+    def __init__(self, message: str, status_code: int = None, response: dict = None):
+        super().__init__(message)
+        self.status_code = status_code
+        self.response = response
+
+
+class ConnectionError(PromptEvalError):
+    """Cannot connect to API"""
+    pass
+
+
+class TimeoutError(PromptEvalError):
+    """Request timed out"""
+    pass