vllm-judge 0.1.4__py3-none-any.whl → 0.1.6__py3-none-any.whl

vllm_judge/__init__.py

@@ -5,7 +5,7 @@ A lightweight library for evaluating text responses using self-hosted language m
 via vLLM's OpenAI-compatible API.
 """
 
-__version__ = "0.1.4"
+__version__ = "0.1.6"
 
 from vllm_judge.judge import Judge
 from vllm_judge.models import (
@@ -17,17 +17,18 @@ from vllm_judge.models import (
     ModelSpecificMetric
 )
 from vllm_judge.templating import TemplateProcessor
-from vllm_judge.metrics import (
+from vllm_judge.builtin_metrics import (
     # General metrics
     HELPFULNESS,
     ACCURACY,
     CLARITY,
     CONCISENESS,
     RELEVANCE,
-    
+    COHERENCE,
     # Safety metrics
     SAFETY,
     TOXICITY,
+    BIAS_DETECTION,
     LLAMA_GUARD_3_SAFETY,
     
     # Code metrics
@@ -61,6 +62,12 @@ from vllm_judge.metrics import (
     PRODUCT_REVIEW_TEMPLATE,
     MEDICAL_INFO_TEMPLATE,
     API_DOCS_TEMPLATE,
+    RAG_EVALUATION_TEMPLATE,
+    AGENT_PERFORMANCE_TEMPLATE,
+
+    # NLP metrics
+    TRANSLATION_QUALITY,
+    SUMMARIZATION_QUALITY,
 
 )
 from vllm_judge.exceptions import (
@@ -91,8 +98,10 @@ __all__ = [
     "CLARITY",
     "CONCISENESS",
     "RELEVANCE",
+    "COHERENCE",
     "SAFETY",
     "TOXICITY",
+    "BIAS_DETECTION",
     "LLAMA_GUARD_3_SAFETY",
     "CODE_QUALITY",
     "CODE_SECURITY",
@@ -112,6 +121,11 @@ __all__ = [
     "PRODUCT_REVIEW_TEMPLATE",
     "MEDICAL_INFO_TEMPLATE",
     "API_DOCS_TEMPLATE",
+    "RAG_EVALUATION_TEMPLATE",
+    "AGENT_PERFORMANCE_TEMPLATE",
+    "TRANSLATION_QUALITY",
+    "SUMMARIZATION_QUALITY",
+
     # Exceptions
     "VLLMJudgeError",
     "ConfigurationError",

vllm_judge/api/__init__.py

@@ -1,6 +1,3 @@
-"""
-API module for vLLM Judge.
-"""
 from vllm_judge.api.server import app, create_app, start_server
 from vllm_judge.api.client import JudgeClient
 from vllm_judge.api.models import (

vllm_judge/api/client.py

@@ -1,6 +1,3 @@
-"""
-HTTP client for vLLM Judge API.
-"""
 import asyncio
 from typing import Union, Dict, List, Optional, Tuple, Any, AsyncIterator
 import httpx

vllm_judge/api/server.py

@@ -1,7 +1,3 @@
-"""
-FastAPI server for vLLM Judge API.
-"""
-import asyncio
 import time
 import uuid
 from datetime import datetime
@@ -14,7 +10,7 @@ import uvicorn
 
 from vllm_judge.judge import Judge
 from vllm_judge.models import EvaluationResult, JudgeConfig
-from vllm_judge.metrics import BUILTIN_METRICS
+from vllm_judge.builtin_metrics import BUILTIN_METRICS
 from vllm_judge.exceptions import VLLMJudgeError
 from vllm_judge.api.models import (
     EvaluateRequest,

vllm_judge/batch.py

@@ -17,7 +17,8 @@ class BatchProcessor:
             max_concurrent: Maximum concurrent requests
         """
         self.judge = judge
-        self.semaphore = asyncio.Semaphore(max_concurrent)
+        self.max_concurrent = max_concurrent
+        self.semaphore = asyncio.Semaphore(self.max_concurrent)
         self.progress_lock = asyncio.Lock()
         self.completed = 0