evalvault 1.62.1__py3-none-any.whl → 1.63.0__py3-none-any.whl

evalvault/adapters/outbound/llm/token_aware_chat.py

@@ -149,7 +149,16 @@ class TokenTrackingAsyncOpenAI(AsyncOpenAI):
 
                 span_attrs = _build_llm_span_attrs(provider_name, kwargs)
                 with instrumentation_span("llm.chat_completion", span_attrs) as span:
-                    response = await inner_self._completions.create(**kwargs)
+                    try:
+                        response = await inner_self._completions.create(**kwargs)
+                    except TypeError as exc:
+                        if "max_completion_tokens" in str(exc):
+                            fallback_kwargs = dict(kwargs)
+                            fallback_kwargs.pop("max_completion_tokens", None)
+                            fallback_kwargs.pop("max_tokens", None)
+                            response = await inner_self._completions.create(**fallback_kwargs)
+                        else:
+                            raise
                     if provider_name == "ollama":
                         _normalize_tool_calls(response, kwargs.get("tools"))
                     # Extract usage from response
@@ -198,6 +207,7 @@ class ThinkingTokenTrackingAsyncOpenAI(TokenTrackingAsyncOpenAI):
 
             async def create(inner_self, **kwargs: Any) -> Any:  # noqa: N805
                 # Ensure 충분한 출력 토큰 확보 (Ollama는 max_tokens를 사용)
+                min_tokens = None
                 if provider_name == "ollama":
                     if "max_tokens" not in kwargs or kwargs["max_tokens"] < 4096:
                         kwargs["max_tokens"] = 16384
@@ -222,7 +232,16 @@ class ThinkingTokenTrackingAsyncOpenAI(TokenTrackingAsyncOpenAI):
 
                 span_attrs = _build_llm_span_attrs(provider_name, kwargs)
                 with instrumentation_span("llm.chat_completion", span_attrs) as span:
-                    response = await inner_self._completions.create(**kwargs)
+                    try:
+                        response = await inner_self._completions.create(**kwargs)
+                    except TypeError as exc:
+                        if "max_completion_tokens" in str(exc):
+                            fallback_kwargs = dict(kwargs)
+                            fallback_kwargs.pop("max_completion_tokens", None)
+                            fallback_kwargs.pop("max_tokens", None)
+                            response = await inner_self._completions.create(**fallback_kwargs)
+                        else:
+                            raise
                     if provider_name == "ollama":
                         _normalize_tool_calls(response, kwargs.get("tools"))
 

evalvault/adapters/outbound/llm/vllm_adapter.py

@@ -12,6 +12,7 @@ from evalvault.adapters.outbound.llm.instructor_factory import create_instructor
 from evalvault.adapters.outbound.llm.token_aware_chat import TokenTrackingAsyncOpenAI
 from evalvault.config.phoenix_support import instrumentation_span, set_span_attributes
 from evalvault.config.settings import Settings
+from evalvault.ports.outbound.llm_port import GenerationOptions
 
 
 class VLLMAdapter(BaseLLMAdapter):
@@ -63,25 +64,46 @@ class VLLMAdapter(BaseLLMAdapter):
         """Get the embedding model name being used."""
         return self._embedding_model_name
 
-    async def agenerate_text(self, prompt: str) -> str:
+    async def agenerate_text(
+        self,
+        prompt: str,
+        *,
+        options: GenerationOptions | None = None,
+    ) -> str:
         """Generate text from a prompt (async)."""
         attrs = {
             "llm.provider": "vllm",
             "llm.model": self._model_name,
             "llm.mode": "async",
         }
+        max_tokens = options.max_tokens if options and options.max_tokens is not None else 8192
+        api_kwargs = {
+            "model": self._model_name,
+            "messages": [{"role": "user", "content": prompt}],
+            "max_completion_tokens": max_tokens,
+        }
+        if options and options.temperature is not None:
+            api_kwargs["temperature"] = options.temperature
+        if options and options.top_p is not None:
+            api_kwargs["top_p"] = options.top_p
+        if options and options.n is not None:
+            api_kwargs["n"] = options.n
+        if options and options.seed is not None:
+            api_kwargs["seed"] = options.seed
         with instrumentation_span("llm.generate_text", attrs) as span:
-            response = await self._client.chat.completions.create(
-                model=self._model_name,
-                messages=[{"role": "user", "content": prompt}],
-                max_completion_tokens=8192,
-            )
+            response = await self._client.chat.completions.create(**api_kwargs)
             content = response.choices[0].message.content or ""
             if span:
                 set_span_attributes(span, {"llm.response.length": len(content)})
         return content
 
-    def generate_text(self, prompt: str, *, json_mode: bool = False) -> str:
+    def generate_text(
+        self,
+        prompt: str,
+        *,
+        json_mode: bool = False,
+        options: GenerationOptions | None = None,
+    ) -> str:
         """Generate text from a prompt (sync)."""
         sync_client = OpenAI(
             base_url=self._settings.vllm_base_url,
@@ -89,11 +111,20 @@ class VLLMAdapter(BaseLLMAdapter):
             timeout=self._settings.vllm_timeout,
         )
 
+        max_tokens = options.max_tokens if options and options.max_tokens is not None else 8192
         api_kwargs: dict = {
             "model": self._model_name,
             "messages": [{"role": "user", "content": prompt}],
-            "max_completion_tokens": 8192,
+            "max_completion_tokens": max_tokens,
         }
+        if options and options.temperature is not None:
+            api_kwargs["temperature"] = options.temperature
+        if options and options.top_p is not None:
+            api_kwargs["top_p"] = options.top_p
+        if options and options.n is not None:
+            api_kwargs["n"] = options.n
+        if options and options.seed is not None:
+            api_kwargs["seed"] = options.seed
         if json_mode:
             api_kwargs["response_format"] = {"type": "json_object"}
 

evalvault/adapters/outbound/nlp/korean/toolkit_factory.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+import logging
+
+from evalvault.ports.outbound.korean_nlp_port import KoreanNLPToolkitPort
+
+logger = logging.getLogger(__name__)
+
+
+def try_create_korean_toolkit() -> KoreanNLPToolkitPort | None:
+    try:
+        from evalvault.adapters.outbound.nlp.korean.toolkit import KoreanNLPToolkit
+    except Exception as exc:
+        logger.debug("Korean toolkit import failed: %s", exc)
+        return None
+    try:
+        return KoreanNLPToolkit()
+    except Exception as exc:
+        logger.debug("Korean toolkit init failed: %s", exc)
+        return None

evalvault/adapters/outbound/report/llm_report_generator.py

@@ -366,6 +366,24 @@ DEFAULT_METRIC_PROMPT = """당신은 RAG 시스템 평가 전문가입니다.
 
 마크다운 형식으로 작성해주세요."""
 
+DEFAULT_METRIC_PROMPT_EN = """You are a RAG evaluation expert.
+
+## Target
+- Metric: {metric_name}
+- Score: {score:.3f} / 1.0
+- Threshold: {threshold:.2f}
+- Status: {status}
+
+## Request
+Provide a Markdown analysis covering:
+
+1. **Current assessment**: what this score implies
+2. **Likely causes**: plausible root causes
+3. **Actionable improvements**: practical steps the team can take
+4. **Expected impact**: anticipated gains per action
+
+Respond in Markdown."""
+
 
 EXECUTIVE_SUMMARY_PROMPT = """당신은 RAG 시스템 성능 개선 전문가입니다. 평가 결과를 분석하고 구체적인 개선 방안을 제시해주세요.
 
@@ -423,6 +441,60 @@ EXECUTIVE_SUMMARY_PROMPT = """당신은 RAG 시스템 성능 개선 전문가입
 
 마크다운 형식으로 작성해주세요."""
 
+EXECUTIVE_SUMMARY_PROMPT_EN = """You are a RAG performance improvement expert. Analyze the evaluation results and propose concrete actions.
+
+Evaluation Results:
+- Dataset: {dataset_name}
+- Model: {model_name}
+- Pass rate: {pass_rate:.1%}
+- Test cases: {total_test_cases}
+
+Metric Scores:
+{metrics_summary}
+
+Analysis Request:
+
+Provide a RAG performance improvement-focused analysis using the structure below:
+
+1) Current Summary (3 sentences)
+- Overall quality level and the most urgent issue
+- Clearly distinguish strong areas vs weak areas
+
+2) Problem Definition
+
+| Problem | Metric | Current | Target | Severity |
+|------|--------|--------|--------|--------|
+| (Specific issue) | (Related metric) | (Score) | (Target) | Critical/High/Medium |
+
+3) Root Cause Analysis
+
+For each problem:
+- Direct cause: immediate cause (e.g., "relevant context is not retrieved")
+- Root cause: structural cause (e.g., "chunk size is too large and dilutes relevance")
+- Verification: how to validate (e.g., "re-run with top_k=10")
+
+4) Solutions
+
+P0 - Immediate (1-3 days)
+For each action:
+- Action: one-line description
+- Implementation: concrete steps (including code/config changes)
+- Expected impact: quantified estimate (e.g., "faithfulness +0.15")
+
+P1 - Short term (1-2 weeks)
+Provide 2-3 actions in the same format
+
+P2 - Mid term (1 month)
+Provide 1-2 actions in the same format
+
+5) Verification Plan
+- How to measure improvement for each action
+- Monitoring indicators to prevent regressions
+
+Note: Do not give abstract advice. All suggestions must be concrete and actionable.
+
+Respond in Markdown."""
+
 SUMMARY_RECOMMENDED_THRESHOLDS = {
     "summary_faithfulness": 0.90,
     "summary_score": 0.85,
@@ -643,11 +715,15 @@ class LLMReportGenerator:
         threshold: float,
     ) -> LLMReportSection:
         """개별 메트릭 분석."""
-        # 프롬프트 선택
-        prompt_template = METRIC_ANALYSIS_PROMPTS.get(metric_name, DEFAULT_METRIC_PROMPT)
+        prompt_template = (
+            DEFAULT_METRIC_PROMPT_EN
+            if self._language == "en"
+            else METRIC_ANALYSIS_PROMPTS.get(metric_name, DEFAULT_METRIC_PROMPT)
+        )
 
-        # 상태 계산
-        status = "통과" if score >= threshold else "미달"
+        status = "pass" if score >= threshold else "fail"
+        if self._language != "en":
+            status = "통과" if score >= threshold else "미달"
 
         prompt = prompt_template.format(
             metric_name=metric_name,
@@ -683,11 +759,19 @@ class LLMReportGenerator:
         for metric, score in metrics_scores.items():
             threshold = thresholds.get(metric, 0.7)
             status = "✅" if score >= threshold else "❌"
-            metrics_lines.append(f"- {metric}: {score:.3f} (임계값: {threshold:.2f}) {status}")
+            if self._language == "en":
+                metrics_lines.append(
+                    f"- {metric}: {score:.3f} (threshold: {threshold:.2f}) {status}"
+                )
+            else:
+                metrics_lines.append(f"- {metric}: {score:.3f} (임계값: {threshold:.2f}) {status}")
 
         metrics_summary = "\n".join(metrics_lines)
 
-        prompt = EXECUTIVE_SUMMARY_PROMPT.format(
+        prompt_template = (
+            EXECUTIVE_SUMMARY_PROMPT_EN if self._language == "en" else EXECUTIVE_SUMMARY_PROMPT
+        )
+        prompt = prompt_template.format(
             dataset_name=run.dataset_name,
             model_name=run.model_name,
             pass_rate=run.pass_rate,