vllm-judge 0.1.0__py3-none-any.whl

vllm_judge/__init__.py

@@ -0,0 +1,120 @@
+"""
+vLLM Judge - LLM-as-a-Judge evaluations for vLLM hosted models.
+
+A lightweight library for evaluating text responses using self-hosted language models
+via vLLM's OpenAI-compatible API.
+"""
+
+__version__ = "0.1.0"
+
+from vllm_judge.judge import Judge
+from vllm_judge.models import (
+    JudgeConfig,
+    EvaluationResult,
+    Metric,
+    BatchResult,
+    TemplateEngine
+)
+from vllm_judge.templating import TemplateProcessor
+from vllm_judge.metrics import (
+    # General metrics
+    HELPFULNESS,
+    ACCURACY,
+    CLARITY,
+    CONCISENESS,
+    RELEVANCE,
+    
+    # Safety metrics
+    SAFETY,
+    TOXICITY,
+    
+    # Code metrics
+    CODE_QUALITY,
+    CODE_SECURITY,
+    
+    # Content metrics
+    CREATIVITY,
+    PROFESSIONALISM,
+    EDUCATIONAL_VALUE,
+    
+    # Comparison metrics
+    PREFERENCE,
+    
+    # Binary metrics
+    APPROPRIATE,
+    FACTUAL,
+    
+    # Domain metrics
+    MEDICAL_ACCURACY,
+    LEGAL_APPROPRIATENESS,
+    
+    # Utility
+    BUILTIN_METRICS,
+
+    # Template metrics
+    EDUCATIONAL_CONTENT_TEMPLATE,
+    CODE_REVIEW_TEMPLATE,
+    CUSTOMER_SERVICE_TEMPLATE,
+    WRITING_QUALITY_TEMPLATE,
+    PRODUCT_REVIEW_TEMPLATE,
+    MEDICAL_INFO_TEMPLATE,
+    API_DOCS_TEMPLATE,
+
+)
+from vllm_judge.exceptions import (
+    VLLMJudgeError,
+    ConfigurationError,
+    ConnectionError,
+    TimeoutError,
+    ParseError,
+    MetricNotFoundError,
+    InvalidInputError,
+    RetryExhaustedError
+)
+
+__all__ = [
+    # Main classes
+    "Judge",
+    "JudgeConfig",
+    "EvaluationResult",
+    "Metric",
+    "BatchResult",
+    "TemplateEngine",
+    "TemplateProcessor",
+
+    # Metrics
+    "HELPFULNESS",
+    "ACCURACY", 
+    "CLARITY",
+    "CONCISENESS",
+    "RELEVANCE",
+    "SAFETY",
+    "TOXICITY",
+    "CODE_QUALITY",
+    "CODE_SECURITY",
+    "CREATIVITY",
+    "PROFESSIONALISM",
+    "EDUCATIONAL_VALUE",
+    "PREFERENCE",
+    "APPROPRIATE",
+    "FACTUAL",
+    "MEDICAL_ACCURACY",
+    "LEGAL_APPROPRIATENESS",
+    "BUILTIN_METRICS",
+    "EDUCATIONAL_CONTENT_TEMPLATE",
+    "CODE_REVIEW_TEMPLATE",
+    "CUSTOMER_SERVICE_TEMPLATE",
+    "WRITING_QUALITY_TEMPLATE",
+    "PRODUCT_REVIEW_TEMPLATE",
+    "MEDICAL_INFO_TEMPLATE",
+    "API_DOCS_TEMPLATE",
+    # Exceptions
+    "VLLMJudgeError",
+    "ConfigurationError",
+    "ConnectionError",
+    "TimeoutError",
+    "ParseError",
+    "MetricNotFoundError",
+    "InvalidInputError",
+    "RetryExhaustedError"
+]

vllm_judge/api/__init__.py

@@ -0,0 +1,39 @@
+"""
+API module for vLLM Judge.
+"""
+from vllm_judge.api.server import app, create_app, start_server
+from vllm_judge.api.client import JudgeClient
+from vllm_judge.api.models import (
+    EvaluateRequest,
+    BatchEvaluateRequest,
+    AsyncBatchRequest,
+    EvaluationResponse,
+    BatchResponse,
+    AsyncBatchResponse,
+    JobStatusResponse,
+    MetricInfo,
+    HealthResponse,
+    ErrorResponse
+)
+
+__all__ = [
+    # Server
+    "app",
+    "create_app",
+    "start_server",
+    
+    # Client
+    "JudgeClient",
+    
+    # Models
+    "EvaluateRequest",
+    "BatchEvaluateRequest",
+    "AsyncBatchRequest",
+    "EvaluationResponse",
+    "BatchResponse",
+    "AsyncBatchResponse",
+    "JobStatusResponse",
+    "MetricInfo",
+    "HealthResponse",
+    "ErrorResponse"
+]

vllm_judge/api/client.py

@@ -0,0 +1,354 @@
+"""
+HTTP client for vLLM Judge API.
+"""
+import asyncio
+from typing import Union, Dict, List, Optional, Tuple, Any, AsyncIterator
+import httpx
+import websockets
+import json
+
+from vllm_judge.models import EvaluationResult, BatchResult
+from vllm_judge.exceptions import VLLMJudgeError, ConnectionError
+from vllm_judge.api.models import (
+    EvaluateRequest,
+    BatchEvaluateRequest,
+    AsyncBatchRequest,
+    MetricInfo
+)
+
+
+class JudgeClient:
+    """HTTP client for vLLM Judge API."""
+    
+    def __init__(
+        self,
+        api_url: str,
+        timeout: float = 30.0,
+        max_retries: int = 3
+    ):
+        """
+        Initialize Judge API client.
+        
+        Args:
+            api_url: Base URL of Judge API server
+            timeout: Request timeout in seconds
+            max_retries: Maximum retry attempts
+        """
+        self.api_url = api_url.rstrip('/')
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self.session = httpx.AsyncClient(
+            base_url=self.api_url,
+            timeout=httpx.Timeout(timeout)
+        )
+    
+    async def __aenter__(self):
+        """Async context manager entry."""
+        return self
+    
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """Async context manager exit."""
+        await self.close()
+    
+    async def close(self):
+        """Close HTTP session."""
+        await self.session.aclose()
+    
+    async def health_check(self) -> Dict[str, Any]:
+        """Check API health status."""
+        try:
+            response = await self.session.get("/health")
+            response.raise_for_status()
+            return response.json()
+        except httpx.HTTPError as e:
+            raise ConnectionError(f"Health check failed: {e}")
+    
+    async def evaluate(
+        self,
+        response: Union[str, Dict[str, str]],
+        criteria: str = None,
+        rubric: Union[str, Dict[Union[int, float], str]] = None,
+        scale: Optional[Tuple[int, int]] = None,
+        metric: str = None,
+        context: str = None,
+        system_prompt: str = None,
+        examples: List[Dict[str, Any]] = None,
+        template_vars: Dict[str, Any] = None,
+        template_engine: str = "format",
+        **kwargs
+    ) -> EvaluationResult:
+        """
+        Perform single evaluation via API.
+        
+        Args:
+            Same as Judge.evaluate() including template support
+            
+        Returns:
+            EvaluationResult
+        """
+        request = EvaluateRequest(
+            response=response,
+            criteria=criteria,
+            rubric=rubric,
+            scale=list(scale) if scale else None,
+            metric=metric,
+            context=context,
+            system_prompt=system_prompt,
+            examples=examples,
+            template_vars=template_vars,
+            template_engine=template_engine
+        )
+        
+        try:
+            api_response = await self.session.post(
+                "/evaluate",
+                json=request.model_dump()
+            )
+            api_response.raise_for_status()
+            data = api_response.json()
+            
+            return EvaluationResult(
+                decision=data["decision"],
+                reasoning=data["reasoning"],
+                score=data.get("score"),
+                metadata=data.get("metadata", {})
+            )
+            
+        except httpx.HTTPStatusError as e:
+            error_detail = e.response.json().get("detail", str(e))
+            raise VLLMJudgeError(f"Evaluation failed: {error_detail}")
+        except httpx.HTTPError as e:
+            raise ConnectionError(f"API request failed: {e}")
+    
+    async def batch_evaluate(
+        self,
+        data: List[Dict[str, Any]],
+        max_concurrent: int = None,
+        default_criteria: str = None,
+        default_metric: str = None,
+        **kwargs
+    ) -> BatchResult:
+        """
+        Perform synchronous batch evaluation.
+        
+        Args:
+            data: List of evaluation inputs
+            max_concurrent: Maximum concurrent requests
+            default_criteria: Default criteria for all evaluations
+            default_metric: Default metric for all evaluations
+            
+        Returns:
+            BatchResult
+        """
+        request = BatchEvaluateRequest(
+            data=data,
+            max_concurrent=max_concurrent,
+            default_criteria=default_criteria,
+            default_metric=default_metric
+        )
+        
+        try:
+            response = await self.session.post(
+                "/batch",
+                json=request.model_dump(),
+                timeout=None  # No timeout for batch operations
+            )
+            response.raise_for_status()
+            data = response.json()
+            
+            # Convert results
+            results = []
+            for r in data["results"]:
+                if "error" in r:
+                    results.append(VLLMJudgeError(r["error"]))
+                else:
+                    results.append(EvaluationResult(
+                        decision=r["decision"],
+                        reasoning=r["reasoning"],
+                        score=r.get("score"),
+                        metadata=r.get("metadata", {})
+                    ))
+            
+            return BatchResult(
+                results=results,
+                total=data["total"],
+                successful=data["successful"],
+                failed=data["failed"],
+                duration_seconds=data["duration_seconds"]
+            )
+            
+        except httpx.HTTPStatusError as e:
+            error_detail = e.response.json().get("detail", str(e))
+            raise VLLMJudgeError(f"Batch evaluation failed: {error_detail}")
+        except httpx.HTTPError as e:
+            raise ConnectionError(f"API request failed: {e}")
+    
+    async def async_batch_evaluate(
+        self,
+        data: List[Dict[str, Any]],
+        callback_url: str = None,
+        max_concurrent: int = None,
+        poll_interval: float = 1.0
+    ) -> BatchResult:
+        """
+        Start async batch evaluation and wait for completion.
+        
+        Args:
+            data: List of evaluation inputs
+            callback_url: Optional callback URL
+            max_concurrent: Maximum concurrent requests
+            poll_interval: Seconds between status checks
+            
+        Returns:
+            BatchResult when complete
+        """
+        # Start async job
+        request = AsyncBatchRequest(
+            data=data,
+            callback_url=callback_url,
+            max_concurrent=max_concurrent
+        )
+        
+        response = await self.session.post(
+            "/batch/async",
+            json=request.model_dump()
+        )
+        response.raise_for_status()
+        job_data = response.json()
+        job_id = job_data["job_id"]
+        
+        # Poll for completion
+        while True:
+            status = await self.get_job_status(job_id)
+            
+            if status["status"] == "completed":
+                return await self.get_job_result(job_id)
+            elif status["status"] == "failed":
+                raise VLLMJudgeError(f"Job failed: {status.get('error', 'Unknown error')}")
+            
+            await asyncio.sleep(poll_interval)
+    
+    async def get_job_status(self, job_id: str) -> Dict[str, Any]:
+        """Get status of async job."""
+        response = await self.session.get(f"/jobs/{job_id}")
+        response.raise_for_status()
+        return response.json()
+    
+    async def get_job_result(self, job_id: str) -> BatchResult:
+        """Get result of completed async job."""
+        response = await self.session.get(f"/jobs/{job_id}/result")
+        response.raise_for_status()
+        data = response.json()
+        
+        # Convert to BatchResult
+        results = []
+        for r in data["results"]:
+            if "error" in r:
+                results.append(VLLMJudgeError(r["error"]))
+            else:
+                results.append(EvaluationResult(
+                    decision=r["decision"],
+                    reasoning=r["reasoning"],
+                    score=r.get("score"),
+                    metadata=r.get("metadata", {})
+                ))
+        
+        return BatchResult(
+            results=results,
+            total=data["total"],
+            successful=data["successful"],
+            failed=data["failed"],
+            duration_seconds=data["duration_seconds"]
+        )
+    
+    async def list_metrics(self) -> List[MetricInfo]:
+        """List all available metrics."""
+        response = await self.session.get("/metrics")
+        response.raise_for_status()
+        return [MetricInfo(**m) for m in response.json()]
+    
+    async def get_metric(self, metric_name: str) -> Dict[str, Any]:
+        """Get details of a specific metric."""
+        response = await self.session.get(f"/metrics/{metric_name}")
+        response.raise_for_status()
+        return response.json()
+    
+    # Convenience methods matching Judge interface
+    async def score(
+        self,
+        criteria: str,
+        response: str,
+        scale: Tuple[int, int] = (1, 10),
+        **kwargs
+    ) -> EvaluationResult:
+        """Quick scoring evaluation."""
+        return await self.evaluate(
+            response=response,
+            criteria=criteria,
+            scale=scale,
+            **kwargs
+        )
+    
+    async def compare(
+        self,
+        response_a: str,
+        response_b: str,
+        criteria: str,
+        **kwargs
+    ) -> EvaluationResult:
+        """Quick comparison evaluation."""
+        return await self.evaluate(
+            response={"a": response_a, "b": response_b},
+            criteria=criteria,
+            **kwargs
+        )
+    
+    async def classify(
+        self,
+        response: str,
+        categories: List[str],
+        criteria: str = None,
+        **kwargs
+    ) -> EvaluationResult:
+        """Quick classification evaluation."""
+        if not criteria:
+            criteria = "appropriate category"
+        
+        rubric = f"Classify into one of these categories: {', '.join(categories)}"
+        
+        return await self.evaluate(
+            response=response,
+            criteria=criteria,
+            rubric=rubric,
+            **kwargs
+        )
+    
+    async def evaluate_streaming(
+        self,
+        response: Union[str, Dict[str, str]],
+        **kwargs
+    ) -> AsyncIterator[str]:
+        """
+        WebSocket-based streaming evaluation.
+        
+        Yields partial results as they arrive.
+        """
+        ws_url = self.api_url.replace("http://", "ws://").replace("https://", "wss://")
+        ws_url = f"{ws_url}/ws/evaluate"
+        
+        async with websockets.connect(ws_url) as websocket:
+            # Send request
+            request_data = {
+                "response": response,
+                **kwargs
+            }
+            await websocket.send(json.dumps(request_data))
+            
+            # Receive result
+            result_data = await websocket.recv()
+            result = json.loads(result_data)
+            
+            if result["status"] == "success":
+                yield json.dumps(result["result"])
+            else:
+                raise VLLMJudgeError(f"Streaming evaluation failed: {result.get('error')}")

vllm_judge/api/models.py

@@ -0,0 +1,157 @@
+from typing import Union, Dict, List, Optional, Any, Tuple
+from pydantic import BaseModel, Field
+from datetime import datetime
+
+
+class EvaluateRequest(BaseModel):
+    """Request model for single evaluation."""
+    response: Union[str, Dict[str, str]] = Field(
+        ..., description="Text to evaluate or dict with 'a' and 'b' for comparison"
+    )
+    criteria: Optional[str] = Field(
+        None, description="What to evaluate for"
+    )
+    rubric: Optional[Union[str, Dict[Union[int, float], str]]] = Field(
+        None, description="Evaluation guide"
+    )
+    scale: Optional[List[int]] = Field(
+        None, description="Numeric scale as [min, max]"
+    )
+    metric: Optional[str] = Field(
+        None, description="Pre-defined metric name"
+    )
+    context: Optional[str] = Field(
+        None, description="Additional context"
+    )
+    system_prompt: Optional[str] = Field(
+        None, description="Custom system prompt"
+    )
+    examples: Optional[List[Dict[str, Any]]] = Field(
+        None, description="Few-shot examples"
+    )
+    template_vars: Optional[Dict[str, Any]] = Field(
+        None, description="Template variables to substitute"
+    )
+    template_engine: Optional[str] = Field(
+        None, description="Template engine to use ('format' or 'jinja2'), default is 'format'"
+    )
+    
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "response": "Python is a high-level programming language...",
+                "criteria": "technical accuracy for {audience}",
+                "template_vars": {"audience": "beginners"},
+                "scale": [1, 10]
+            }
+        }
+
+
+class BatchEvaluateRequest(BaseModel):
+    """Request model for batch evaluation."""
+    data: List[Dict[str, Any]] = Field(
+        ..., description="List of evaluation inputs"
+    )
+    max_concurrent: Optional[int] = Field(
+        None, description="Maximum concurrent requests"
+    )
+    default_criteria: Optional[str] = Field(
+        None, description="Default criteria for all evaluations"
+    )
+    default_metric: Optional[str] = Field(
+        None, description="Default metric for all evaluations"
+    )
+
+
+class AsyncBatchRequest(BaseModel):
+    """Request model for async batch evaluation."""
+    data: List[Dict[str, Any]] = Field(
+        ..., description="List of evaluation inputs"
+    )
+    callback_url: Optional[str] = Field(
+        None, description="URL to POST results when complete"
+    )
+    max_concurrent: Optional[int] = Field(
+        None, description="Maximum concurrent requests"
+    )
+
+
+class EvaluationResponse(BaseModel):
+    """Response model for evaluation results."""
+    decision: Union[str, bool, int, float]
+    reasoning: str
+    score: Optional[float]
+    metadata: Dict[str, Any] = {}
+    
+    # API-specific fields
+    evaluation_id: Optional[str] = None
+    timestamp: Optional[datetime] = None
+    duration_ms: Optional[int] = None
+
+
+class BatchResponse(BaseModel):
+    """Response model for batch results."""
+    total: int
+    successful: int
+    failed: int
+    success_rate: float
+    duration_seconds: float
+    results: List[Union[EvaluationResponse, Dict[str, str]]]
+
+
+class AsyncBatchResponse(BaseModel):
+    """Response model for async batch initiation."""
+    job_id: str
+    status: str = "pending"
+    total_items: int
+    created_at: datetime
+    estimated_duration_seconds: Optional[float] = None
+
+
+class JobStatusResponse(BaseModel):
+    """Response model for job status."""
+    job_id: str
+    status: str  # "pending", "running", "completed", "failed"
+    progress: Dict[str, int]  # {"completed": 50, "total": 100}
+    created_at: datetime
+    started_at: Optional[datetime] = None
+    completed_at: Optional[datetime] = None
+    result_url: Optional[str] = None
+    error: Optional[str] = None
+
+
+class MetricInfo(BaseModel):
+    """Information about a metric."""
+    name: str
+    criteria: str
+    has_scale: bool
+    scale: Optional[Tuple[int, int]] = None
+    has_rubric: bool
+    rubric_type: Optional[str] = None  # "string" or "dict"
+    has_examples: bool
+    example_count: int = 0
+    has_system_prompt: bool
+    has_template_vars: bool = False
+    template_vars: Optional[Dict[str, Any]] = None
+    required_vars: Optional[List[str]] = None
+    template_engine: Optional[str] = None
+
+
+class HealthResponse(BaseModel):
+    """Health check response."""
+    status: str = "healthy"
+    version: str
+    model: str
+    base_url: str
+    uptime_seconds: float
+    total_evaluations: int
+    active_connections: int
+    metrics_available: int
+
+
+class ErrorResponse(BaseModel):
+    """Error response model."""
+    error: str
+    detail: Optional[str] = None
+    code: Optional[str] = None
+    timestamp: datetime = Field(default_factory=datetime.utcnow)