vllm-judge 0.1.5__tar.gz → 0.1.6__tar.gz

{vllm_judge-0.1.5 → vllm_judge-0.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vllm_judge
-Version: 0.1.5
+Version: 0.1.6
 Summary: LLM-as-a-Judge evaluations for vLLM hosted models
 Author: TrustyAI team
 Author-email: Sai Chandra Pandraju <saichandrapandraju@gmail.com>

{vllm_judge-0.1.5 → vllm_judge-0.1.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "vllm_judge"
-version = "0.1.5"
+version = "0.1.6"
 description = "LLM-as-a-Judge evaluations for vLLM hosted models"
 readme = "README.md"
 authors = [

{vllm_judge-0.1.5 → vllm_judge-0.1.6}/src/vllm_judge/__init__.py

@@ -5,7 +5,7 @@ A lightweight library for evaluating text responses using self-hosted language m
 via vLLM's OpenAI-compatible API.
 """
 
-__version__ = "0.1.5"
+__version__ = "0.1.6"
 
 from vllm_judge.judge import Judge
 from vllm_judge.models import (
@@ -17,7 +17,7 @@ from vllm_judge.models import (
     ModelSpecificMetric
 )
 from vllm_judge.templating import TemplateProcessor
-from vllm_judge.metrics import (
+from vllm_judge.builtin_metrics import (
     # General metrics
     HELPFULNESS,
     ACCURACY,

{vllm_judge-0.1.5 → vllm_judge-0.1.6}/src/vllm_judge/api/__init__.py

@@ -1,6 +1,3 @@
-"""
-API module for vLLM Judge.
-"""
 from vllm_judge.api.server import app, create_app, start_server
 from vllm_judge.api.client import JudgeClient
 from vllm_judge.api.models import (

{vllm_judge-0.1.5 → vllm_judge-0.1.6}/src/vllm_judge/api/client.py

@@ -1,6 +1,3 @@
-"""
-HTTP client for vLLM Judge API.
-"""
 import asyncio
 from typing import Union, Dict, List, Optional, Tuple, Any, AsyncIterator
 import httpx

{vllm_judge-0.1.5 → vllm_judge-0.1.6}/src/vllm_judge/api/server.py

@@ -1,7 +1,3 @@
-"""
-FastAPI server for vLLM Judge API.
-"""
-import asyncio
 import time
 import uuid
 from datetime import datetime
@@ -14,7 +10,7 @@ import uvicorn
 
 from vllm_judge.judge import Judge
 from vllm_judge.models import EvaluationResult, JudgeConfig
-from vllm_judge.metrics import BUILTIN_METRICS
+from vllm_judge.builtin_metrics import BUILTIN_METRICS
 from vllm_judge.exceptions import VLLMJudgeError
 from vllm_judge.api.models import (
     EvaluateRequest,

{vllm_judge-0.1.5 → vllm_judge-0.1.6}/src/vllm_judge/batch.py

@@ -17,7 +17,8 @@ class BatchProcessor:
             max_concurrent: Maximum concurrent requests
         """
         self.judge = judge
-        self.semaphore = asyncio.Semaphore(max_concurrent)
+        self.max_concurrent = max_concurrent
+        self.semaphore = asyncio.Semaphore(self.max_concurrent)
         self.progress_lock = asyncio.Lock()
         self.completed = 0
     

vllm_judge-0.1.5/src/vllm_judge/metrics.py → vllm_judge-0.1.6/src/vllm_judge/builtin_metrics.py

@@ -1,6 +1,6 @@
 from typing import Dict
 from vllm_judge.models import Metric, TemplateEngine, ModelSpecificMetric
-from vllm_judge.utils import parse_llama_guard_3
+from vllm_judge.parsers import parse_llama_guard_3
 
 # Registry for built-in metrics
 BUILTIN_METRICS: Dict[str, Metric] = {}

{vllm_judge-0.1.5 → vllm_judge-0.1.6}/src/vllm_judge/cli.py

@@ -1,6 +1,3 @@
-"""
-Command-line interface for vLLM Judge.
-"""
 import asyncio
 import json
 import sys
@@ -8,10 +5,9 @@ from typing import Optional
 import click
 
 from vllm_judge import Judge
-from vllm_judge.models import JudgeConfig
 from vllm_judge.api.server import start_server as start_api_server
 from vllm_judge.api.client import JudgeClient
-from vllm_judge.metrics import BUILTIN_METRICS
+from vllm_judge.builtin_metrics import BUILTIN_METRICS
 
 
 @click.group()

{vllm_judge-0.1.5 → vllm_judge-0.1.6}/src/vllm_judge/client.py

@@ -126,12 +126,8 @@ class VLLMClient:
             "messages": messages,
             "temperature": self.config.temperature,
             "max_tokens": self.config.max_tokens,
-            # "top_p": self.config.top_p,
         }
-        
-        # # Request JSON response format if supported
-        # if self.config.temperature < 0.2:  # Only for low temperature
-        #     request_data["response_format"] = {"type": "json_object"}
+
         
         try:
             response = await self._request_with_retry(
@@ -172,7 +168,6 @@ class VLLMClient:
             "prompt": prompt,
             "temperature": self.config.temperature,
             "max_tokens": self.config.max_tokens,
-            # "top_p": self.config.top_p,
         }
         
         try:

{vllm_judge-0.1.5 → vllm_judge-0.1.6}/src/vllm_judge/judge.py

@@ -4,9 +4,9 @@ from typing import Union, Dict, List, Optional, Tuple, Any, Callable
 
 from vllm_judge.models import JudgeConfig, EvaluationResult, Metric, BatchResult, TemplateEngine, ModelSpecificMetric
 from vllm_judge.client import VLLMClient
-from vllm_judge.prompts import PromptBuilder
+from vllm_judge.prompt_builder import PromptBuilder
 from vllm_judge.batch import BatchProcessor
-from vllm_judge.metrics import BUILTIN_METRICS
+from vllm_judge.builtin_metrics import BUILTIN_METRICS
 from vllm_judge.templating import TemplateProcessor
 from vllm_judge.exceptions import (
     ParseError,

{vllm_judge-0.1.5 → vllm_judge-0.1.6}/src/vllm_judge/models.py

@@ -59,7 +59,6 @@ class JudgeConfig(BaseModel):
     # Model parameters
     temperature: float = Field(0.0, description="Sampling temperature")
     max_tokens: int = Field(256, description="Maximum tokens in response")
-    # top_p: float = Field(0.95, description="Top-p sampling")
     
     # Batch settings
     max_concurrent: int = Field(50, description="Maximum concurrent requests")

vllm_judge-0.1.5/src/vllm_judge/prompts.py → vllm_judge-0.1.6/src/vllm_judge/prompt_builder.py

@@ -1,5 +1,5 @@
 from typing import List, Dict, Union, Optional, Tuple, Any
-
+import json
 
 class PromptBuilder:
     """Builds prompts for evaluation requests."""
@@ -35,24 +35,36 @@ class PromptBuilder:
         """
         # Detect evaluation type
         is_comparison = isinstance(content, dict) and "a" in content and "b" in content
+
+        output_format = """
+# Output Format:
+
+The JSON object MUST have exactly these three fields:
+
+1. decision: (String | Boolean) This decision label should clearly state your main finding. This could be a string representing a specific class (eg., PASS, FAIL, CORRECT, INCORRECT, etc.) or a boolean value (true or false). If user provided a rubric, you should use the rubric to determine the decision label.
+2. score: (Number | null) A numerical score for the evaluation. If scoring is requested, provide the score as a number. If scoring is NOT requested or is not applicable for the specific task, you MUST use the value null for this field.
+3. reasoning: (String) A concise explanation justifying your decision and score (if a score was provided). This reasoning must directly and logically support your evaluation and refer to the specific evaluation criteria.
+
+The JSON object MUST be well-formed and adhere strictly to the following structure:
+
+{
+    "decision": <your judgment - string|boolean>,
+    "reasoning": <concise explanation of your judgment - string>,
+    "score": <numeric score if requested, otherwise null - number|null>
+}
+        """
         
         # System message
         if not system_prompt:
-            # TODO: Add more detailed system prompts
-            system_prompt = "You are an impartial judge and expert evaluator "
-            if is_comparison:
-                system_prompt+="comparing responses objectively."
-            else:
-                system_prompt+="providing objective assessments."
-        
-        # Output format instructions
-        system_prompt+="\nYou must respond in JSON format:\n"
-        system_prompt+="""{
-    "decision": <your judgment - string|boolean>,
-    "reasoning": "<concise explanation of your judgment>",
-    "score": <numeric score if requested, otherwise null>
-}"""
-        system_prompt+="\nDo not include any text in your response except for the JSON object."
+            system_prompt = """You are an impartial judge and expert evaluator. Your task is to evaluate the provided content based on the specific evaluation criteria and rubric.
+# Key Instructions:
+1. Your evaluation must be objective, consistent, and based solely on the specified criteria. Do not let your own opinions or biases interfere.
+2. Focus exclusively on quality assessment. 
+3. Do not be influenced by the length of the responses unless response length is explicitly relevant to the specified evaluation criteria (e.g., a task assessing conciseness or verbosity).
+4. Your entire response MUST be a single, valid JSON object and nothing else. Do not include any text or conversational filler before or after this JSON object.
+
+"""
+        system_prompt += output_format
         
         # Build user message
         user_content = PromptBuilder._build_user_prompt(
@@ -93,30 +105,30 @@ class PromptBuilder:
             parts.append(f'"{input}"')
             parts.append("")
         
+        parts.append("## Content to evaluate:")
+        if is_comparison:
+            parts.append(f"**Response A:**\n{content['a']}")
+            parts.append(f"**Response B:**\n{content['b']}")
+        else:
+            parts.append(content)
+        
+        parts.append("## Evaluation Criteria:")
+        
         # Task description
         if is_comparison:
-            if input:
-                parts.append(f"Compare how well these two responses address the input for: {criteria}")
-            else:
-                parts.append(f"Compare these two responses based on: {criteria}")
+            parts.append(f"Compare the two responses based on: {criteria}")
             if context:
                 parts.append(f"\nContext: {context}")
-            parts.append(f"\nResponse A:\n{content['a']}")
-            parts.append(f"\nResponse B:\n{content['b']}")
         else:
-            if input:
-                parts.append(f"Evaluate how well this content addresses the input for: {criteria}")
-            else:
-                parts.append(f"Evaluate the following content based on: {criteria}")
+            parts.append(f"Evaluate the content based on: {criteria}")
             if context:
                 parts.append(f"\nContext: {context}")
-            parts.append(f"\nContent to evaluate:\n{content}")
         
-        parts.append(f"\nYou must return a decision label/class (your judgement) for the `decision` field and a concise explanation for the `reasoning` field.")
+        parts.append(f"\nYou must return a decision label/class (your main judgement) for the `decision` field and a concise explanation for the `reasoning` field in the JSON object.")
 
         # Add scale and rubric
         if scale:
-            parts.append(f"\nIn addition to these, provide a score from {scale[0]} to {scale[1]}")
+            parts.append(f"In addition to these, provide a score from {scale[0]} to {scale[1]}")
             
             if isinstance(rubric, dict):
                 parts.append("\nScoring guide:")
@@ -127,14 +139,15 @@ class PromptBuilder:
             elif rubric:
                 parts.append(f"\nEvaluation guide: {rubric}")
         elif rubric:
+            parts.append("\nIn addition to these, provide a score if required by the following evaluation guide.")
             parts.append(f"\nEvaluation guide: {rubric}")
         
         # Add examples if provided
         if examples:
             parts.append("\nExample evaluations:")
-            for i, ex in enumerate(examples, 1):
-                parts.append(f"\nExample {i}:")
-                
+            for i, ex in enumerate(examples):
+                parts.append(f"Example {i+1}:")
+                parts.append("Request:")
                 # Handle different example formats
                 if "input" in ex:
                     parts.append(f"Input: {ex['input']}")
@@ -143,17 +156,24 @@ class PromptBuilder:
                 elif "text" in ex:
                     parts.append(f"Text: {ex['text']}")
                 
-                if "decision" in ex:
-                    parts.append(f"Decision: {ex['decision']}")
+                parts.append("Response:")
+                
+                response = {}
+                if "decision" not in ex or ex["decision"] is None or ex["decision"] == "":
+                    raise ValueError("Example must include a decision field")
+                
+                response["decision"] = ex["decision"]
                 if "score" in ex:
-                    parts.append(f"Score: {ex['score']}")
+                    response["score"] = ex["score"]
                 
                 if "reasoning" in ex:
-                    parts.append(f"Reasoning: {ex['reasoning']}")
+                    response["reasoning"] = ex["reasoning"]
+                
+                parts.append(json.dumps(response))
         
         # Add any additional instructions
         if kwargs.get("additional_instructions"):
-            parts.append(f"\nAdditional instructions: {kwargs['additional_instructions']}")
+            parts.append(f"Additional instructions: {kwargs['additional_instructions']}")
 
         # Output format instructions
         parts.append("\nYou must respond in JSON format:")

{vllm_judge-0.1.5 → vllm_judge-0.1.6}/src/vllm_judge.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vllm_judge
-Version: 0.1.5
+Version: 0.1.6
 Summary: LLM-as-a-Judge evaluations for vLLM hosted models
 Author: TrustyAI team
 Author-email: Sai Chandra Pandraju <saichandrapandraju@gmail.com>

{vllm_judge-0.1.5 → vllm_judge-0.1.6}/src/vllm_judge.egg-info/SOURCES.txt

@@ -2,15 +2,15 @@ README.md
 pyproject.toml
 src/vllm_judge/__init__.py
 src/vllm_judge/batch.py
+src/vllm_judge/builtin_metrics.py
 src/vllm_judge/cli.py
 src/vllm_judge/client.py
 src/vllm_judge/exceptions.py
 src/vllm_judge/judge.py
-src/vllm_judge/metrics.py
 src/vllm_judge/models.py
-src/vllm_judge/prompts.py
+src/vllm_judge/parsers.py
+src/vllm_judge/prompt_builder.py
 src/vllm_judge/templating.py
-src/vllm_judge/utils.py
 src/vllm_judge.egg-info/PKG-INFO
 src/vllm_judge.egg-info/SOURCES.txt
 src/vllm_judge.egg-info/dependency_links.txt
@@ -20,4 +20,14 @@ src/vllm_judge.egg-info/top_level.txt
 src/vllm_judge/api/__init__.py
 src/vllm_judge/api/client.py
 src/vllm_judge/api/models.py
-src/vllm_judge/api/server.py
+src/vllm_judge/api/server.py
+tests/test_batch.py
+tests/test_builtin_metrics.py
+tests/test_exceptions.py
+tests/test_integration.py
+tests/test_judge.py
+tests/test_judge_client.py
+tests/test_models.py
+tests/test_prompt_builder.py
+tests/test_templating.py
+tests/test_vllm_client.py

vllm_judge-0.1.6/tests/test_batch.py

@@ -0,0 +1,142 @@
+import pytest
+import asyncio
+from unittest.mock import AsyncMock, Mock
+from vllm_judge.batch import BatchProcessor
+from vllm_judge.models import EvaluationResult, BatchResult
+
+
+class TestBatchProcessor:
+    """Test BatchProcessor functionality."""
+    
+    @pytest.fixture
+    def mock_judge(self):
+        """Mock Judge for batch processing tests."""
+        judge = Mock()
+        judge.evaluate = AsyncMock()
+        judge.evaluate.return_value = EvaluationResult(
+            decision="GOOD", reasoning="Test reasoning"
+        )
+        return judge
+    
+    async def test_batch_processor_init(self, mock_judge):
+        """Test BatchProcessor initialization."""
+        processor = BatchProcessor(mock_judge, max_concurrent=10)
+        assert processor.judge == mock_judge
+        assert processor.max_concurrent == 10
+    
+    async def test_batch_process_success(self, mock_judge):
+        """Test successful batch processing."""
+        processor = BatchProcessor(mock_judge, max_concurrent=2)
+        
+        data = [
+            {"content": "Text 1", "criteria": "quality"},
+            {"content": "Text 2", "criteria": "accuracy"},
+            {"content": "Text 3", "criteria": "clarity"}
+        ]
+        
+        result = await processor.process(data)
+        
+        assert isinstance(result, BatchResult)
+        assert result.total == 3
+        assert result.successful == 3
+        assert result.failed == 0
+        assert len(result.results) == 3
+        
+        # Check that all results are EvaluationResult instances
+        for res in result.results:
+            assert isinstance(res, EvaluationResult)
+    
+    async def test_batch_process_with_failures(self, mock_judge):
+        """Test batch processing with some failures."""
+        processor = BatchProcessor(mock_judge, max_concurrent=2)
+        
+        # Make the second call fail
+        mock_judge.evaluate.side_effect = [
+            EvaluationResult(decision="GOOD", reasoning="Success"),
+            Exception("Evaluation failed"),
+            EvaluationResult(decision="OK", reasoning="Success")
+        ]
+        
+        data = [
+            {"content": "Text 1", "criteria": "quality"},
+            {"content": "Text 2", "criteria": "accuracy"},  # This will fail
+            {"content": "Text 3", "criteria": "clarity"}
+        ]
+        
+        result = await processor.process(data)
+        
+        assert result.total == 3
+        assert result.successful == 2
+        assert result.failed == 1
+        assert result.success_rate == 2/3
+        
+        # Check failures
+        failures = result.get_failures()
+        assert len(failures) == 1
+        assert failures[0][0] == 1  # Second item (index 1) failed
+        assert isinstance(failures[0][1], Exception)
+    
+    async def test_batch_process_with_progress_callback(self, mock_judge):
+        """Test batch processing with progress callback."""
+        processor = BatchProcessor(mock_judge, max_concurrent=1)
+        
+        progress_calls = []
+        def progress_callback(completed, total):
+            progress_calls.append((completed, total))
+        
+        data = [
+            {"content": "Text 1", "criteria": "quality"},
+            {"content": "Text 2", "criteria": "accuracy"}
+        ]
+        
+        await processor.process(data, progress_callback=progress_callback)
+        
+        # Should have recorded progress
+        assert len(progress_calls) >= 2
+        assert progress_calls[-1] == (2, 2)  # Final call should be (completed, total)
+    
+    async def test_batch_process_default_kwargs(self, mock_judge):
+        """Test batch processing with default kwargs."""
+        processor = BatchProcessor(mock_judge, max_concurrent=1)
+        
+        data = [
+            {"content": "Text 1"},  # No criteria specified
+            {"content": "Text 2", "criteria": "specific_criteria"}  # Override
+        ]
+        
+        default_kwargs = {"criteria": "default_criteria"}
+        
+        await processor.process(data, **default_kwargs)
+        
+        # Check that evaluate was called with merged kwargs
+        calls = mock_judge.evaluate.call_args_list
+        assert len(calls) == 2
+        
+        # First call should use default criteria
+        assert calls[0][1]["criteria"] == "default_criteria"
+        # Second call should use specific criteria (override)
+        assert calls[1][1]["criteria"] == "specific_criteria"
+    
+    async def test_batch_process_concurrency_limit(self, mock_judge):
+        """Test that concurrency limit is respected."""
+        max_concurrent = 2
+        processor = BatchProcessor(mock_judge, max_concurrent=max_concurrent)
+        
+        call_times = []
+        
+        async def mock_evaluate(**kwargs):
+            call_times.append(asyncio.get_event_loop().time())
+            await asyncio.sleep(0.1)  # Simulate some processing time
+            return EvaluationResult(decision="GOOD", reasoning="Test")
+        
+        mock_judge.evaluate.side_effect = mock_evaluate
+        
+        data = [{"content": f"Text {i}", "criteria": "test"} for i in range(5)]
+        
+        start_time = asyncio.get_event_loop().time()
+        await processor.process(data)
+        end_time = asyncio.get_event_loop().time()
+        
+        # With max_concurrent=2 and 0.1s per call, 5 calls should take at least 0.3s
+        # (first 2 in parallel, then next 2 in parallel, then last 1)
+        assert end_time - start_time >= 0.25  # Allow some margin for timing

vllm_judge-0.1.6/tests/test_builtin_metrics.py

@@ -0,0 +1,71 @@
+from vllm_judge.builtin_metrics import (
+    HELPFULNESS, ACCURACY, SAFETY, CODE_QUALITY,
+    BUILTIN_METRICS, LLAMA_GUARD_3_SAFETY
+)
+from vllm_judge.models import Metric, ModelSpecificMetric
+
+
+class TestBuiltinMetrics:
+    """Test built-in metrics."""
+    
+    def test_builtin_metrics_are_metrics(self):
+        """Test that built-in metrics are Metric instances."""
+        assert isinstance(HELPFULNESS, Metric)
+        assert isinstance(ACCURACY, Metric)
+        assert isinstance(SAFETY, Metric)
+        assert isinstance(CODE_QUALITY, Metric)
+    
+    def test_builtin_metrics_have_names(self):
+        """Test that built-in metrics have proper names."""
+        assert HELPFULNESS.name.upper() == "HELPFULNESS"
+        assert ACCURACY.name.upper() == "ACCURACY" 
+        assert SAFETY.name.upper() == "SAFETY"
+        assert CODE_QUALITY.name.upper() == "CODE_QUALITY"
+    
+    def test_builtin_metrics_have_criteria(self):
+        """Test that built-in metrics have criteria defined."""
+        assert HELPFULNESS.criteria is not None
+        assert len(HELPFULNESS.criteria) > 0
+        
+        assert ACCURACY.criteria is not None
+        assert len(ACCURACY.criteria) > 0
+    
+    def test_builtin_metrics_dict(self):
+        """Test BUILTIN_METRICS dictionary."""
+        assert isinstance(BUILTIN_METRICS, dict)
+        assert "HELPFULNESS".lower() in BUILTIN_METRICS
+        assert "ACCURACY".lower() in BUILTIN_METRICS
+        assert BUILTIN_METRICS["HELPFULNESS".lower()] == HELPFULNESS
+    
+    def test_model_specific_metrics(self):
+        """Test model-specific metrics like Llama Guard."""
+        assert isinstance(LLAMA_GUARD_3_SAFETY, ModelSpecificMetric)
+        assert LLAMA_GUARD_3_SAFETY.name.upper() == "LLAMA_GUARD_3_SAFETY"
+        assert LLAMA_GUARD_3_SAFETY.model_pattern is not None
+        assert LLAMA_GUARD_3_SAFETY.parser_func is not None
+    
+    def test_metrics_with_scales(self):
+        """Test metrics that have defined scales."""
+        # Some metrics should have scales defined
+        scale_metrics = [m for m in BUILTIN_METRICS.values() if m.scale is not None]
+        assert len(scale_metrics) > 0
+        
+        # Check scale format
+        for metric in scale_metrics:
+            assert isinstance(metric.scale, tuple)
+            assert len(metric.scale) == 2
+            assert metric.scale[0] < metric.scale[1]
+    
+    def test_template_metrics(self):
+        """Test metrics that use templates."""
+        from vllm_judge.builtin_metrics import (
+            EDUCATIONAL_CONTENT_TEMPLATE,
+            CODE_REVIEW_TEMPLATE
+        )
+        
+        assert isinstance(EDUCATIONAL_CONTENT_TEMPLATE, Metric)
+        assert isinstance(CODE_REVIEW_TEMPLATE, Metric)
+        
+        # These should have template variables
+        assert len(EDUCATIONAL_CONTENT_TEMPLATE.required_vars) > 0
+        assert len(CODE_REVIEW_TEMPLATE.required_vars) > 0

vllm_judge-0.1.6/tests/test_exceptions.py

@@ -0,0 +1,80 @@
+from vllm_judge.exceptions import (
+    VLLMJudgeError,
+    ConfigurationError,
+    ConnectionError,
+    TimeoutError,
+    ParseError,
+    MetricNotFoundError,
+    InvalidInputError,
+    RetryExhaustedError
+)
+
+
+class TestExceptions:
+    """Test custom exceptions."""
+    
+    def test_base_exception(self):
+        """Test base VLLMJudgeError."""
+        error = VLLMJudgeError("Base error")
+        assert str(error) == "Base error"
+        assert isinstance(error, Exception)
+    
+    def test_configuration_error(self):
+        """Test ConfigurationError."""
+        error = ConfigurationError("Invalid config")
+        assert str(error) == "Invalid config"
+        assert isinstance(error, VLLMJudgeError)
+    
+    def test_connection_error(self):
+        """Test ConnectionError."""
+        error = ConnectionError("Cannot connect to server")
+        assert str(error) == "Cannot connect to server"
+        assert isinstance(error, VLLMJudgeError)
+    
+    def test_timeout_error(self):
+        """Test TimeoutError."""
+        error = TimeoutError("Request timed out")
+        assert str(error) == "Request timed out"
+        assert isinstance(error, VLLMJudgeError)
+    
+    def test_parse_error(self):
+        """Test ParseError with raw response."""
+        raw_response = "Invalid JSON response"
+        error = ParseError("Cannot parse response", raw_response=raw_response)
+        
+        assert str(error) == "Cannot parse response"
+        assert error.raw_response == raw_response
+        assert isinstance(error, VLLMJudgeError)
+    
+    def test_parse_error_without_raw_response(self):
+        """Test ParseError without raw response."""
+        error = ParseError("Cannot parse response")
+        assert str(error) == "Cannot parse response"
+        assert error.raw_response is None
+    
+    def test_metric_not_found_error(self):
+        """Test MetricNotFoundError."""
+        error = MetricNotFoundError("Metric 'unknown' not found")
+        assert str(error) == "Metric 'unknown' not found"
+        assert isinstance(error, VLLMJudgeError)
+    
+    def test_invalid_input_error(self):
+        """Test InvalidInputError."""
+        error = InvalidInputError("Invalid input parameters")
+        assert str(error) == "Invalid input parameters"
+        assert isinstance(error, VLLMJudgeError)
+    
+    def test_retry_exhausted_error(self):
+        """Test RetryExhaustedError."""
+        last_error = Exception("Last attempt failed")
+        error = RetryExhaustedError("All retries failed", last_error=last_error)
+        
+        assert str(error) == "All retries failed"
+        assert error.last_error == last_error
+        assert isinstance(error, VLLMJudgeError)
+    
+    def test_retry_exhausted_error_without_last_error(self):
+        """Test RetryExhaustedError without last error."""
+        error = RetryExhaustedError("All retries failed")
+        assert str(error) == "All retries failed"
+        assert error.last_error is None