evalvault 1.62.1__py3-none-any.whl → 1.63.0__py3-none-any.whl

evalvault/domain/services/intent_classifier.py

@@ -157,6 +157,79 @@ class IntentKeywordRegistry:
             "추이",
             "history",
         }
+        self._keywords[AnalysisIntent.ANALYZE_STATISTICAL] = {
+            "통계",
+            "statistical",
+            "statistics",
+            "평균",
+            "mean",
+            "median",
+            "분산",
+            "variance",
+            "표준편차",
+            "std",
+        }
+        self._keywords[AnalysisIntent.ANALYZE_NLP] = {
+            "nlp",
+            "언어",
+            "텍스트",
+            "text",
+            "문장",
+            "sentence",
+            "키워드",
+            "keyword",
+            "토픽",
+            "topic",
+        }
+        self._keywords[AnalysisIntent.ANALYZE_CAUSAL] = {
+            "인과",
+            "causal",
+            "cause",
+            "effect",
+            "원인",
+            "영향",
+            "intervention",
+        }
+        self._keywords[AnalysisIntent.ANALYZE_NETWORK] = {
+            "네트워크",
+            "network",
+            "graph",
+            "그래프",
+            "연결",
+            "centrality",
+        }
+        self._keywords[AnalysisIntent.ANALYZE_PLAYBOOK] = {
+            "playbook",
+            "플레이북",
+            "규칙",
+            "rule",
+            "추천",
+            "recommendation",
+        }
+        self._keywords[AnalysisIntent.DETECT_ANOMALIES] = {
+            "이상",
+            "anomaly",
+            "anomalies",
+            "outlier",
+            "이상치",
+            "detect",
+        }
+        self._keywords[AnalysisIntent.FORECAST_PERFORMANCE] = {
+            "예측",
+            "forecast",
+            "predict",
+            "projection",
+            "미래",
+            "future",
+        }
+        self._keywords[AnalysisIntent.GENERATE_HYPOTHESES] = {
+            "가설",
+            "hypothesis",
+            "hypotheses",
+            "실험",
+            "experiment",
+            "검증",
+        }
         self._keywords[AnalysisIntent.BENCHMARK_RETRIEVAL] = {
             "벤치마크",
             "benchmark",

evalvault/domain/services/pipeline_template_registry.py

@@ -81,6 +81,9 @@ class PipelineTemplateRegistry:
         """의도에 대응하는 파이프라인 템플릿 조회."""
         return self._templates.get(intent)
 
+    def list_all(self) -> list[tuple[AnalysisIntent, AnalysisPipeline]]:
+        return list(self._templates.items())
+
     # =========================================================================
     # Verification Templates
     # =========================================================================

evalvault/domain/services/prompt_candidate_service.py

@@ -0,0 +1,117 @@
+"""Candidate collection service for prompt suggestions."""
+
+from __future__ import annotations
+
+from dataclasses import replace
+from pathlib import Path
+from typing import Any
+
+from evalvault.domain.entities.prompt_suggestion import PromptCandidate
+
+
+class PromptCandidateService:
+    """Build prompt candidates from manual and auto sources."""
+
+    def build_candidates(
+        self,
+        *,
+        base_prompt: str,
+        role: str,
+        metrics: list[str],
+        manual_prompts: list[str],
+        manual_prompt_files: list[Path],
+        auto: bool,
+        auto_count: int,
+        metadata: dict[str, Any] | None = None,
+    ) -> list[PromptCandidate]:
+        base_metadata = metadata or {}
+        candidates: list[PromptCandidate] = []
+        seen: set[str] = set()
+
+        def add_candidate(
+            content: str, *, source: str, extra: dict[str, Any] | None = None
+        ) -> None:
+            normalized = content.strip()
+            if not normalized:
+                return
+            if normalized in seen:
+                return
+            candidate_metadata = {**base_metadata, **(extra or {})}
+            candidates.append(
+                PromptCandidate(
+                    candidate_id="",
+                    source=source,
+                    content=normalized,
+                    metadata=candidate_metadata,
+                )
+            )
+            seen.add(normalized)
+
+        for index, prompt in enumerate(manual_prompts):
+            add_candidate(prompt, source="manual", extra={"manual_index": index})
+
+        for path in manual_prompt_files:
+            for line_number, line in enumerate(self._read_prompt_file(path), start=1):
+                add_candidate(
+                    line,
+                    source="manual",
+                    extra={"file_path": str(path), "file_line": line_number},
+                )
+
+        if auto and auto_count > 0:
+            for name, content in self._build_auto_variants(
+                base_prompt=base_prompt,
+                role=role,
+                metrics=metrics,
+                auto_count=auto_count,
+            ):
+                add_candidate(
+                    content,
+                    source="auto",
+                    extra={"variant": name, "generator": "template"},
+                )
+
+        return [
+            replace(candidate, candidate_id=f"cand-{index:03d}")
+            for index, candidate in enumerate(candidates, start=1)
+        ]
+
+    def _read_prompt_file(self, path: Path) -> list[str]:
+        lines = path.read_text(encoding="utf-8").splitlines()
+        return [line.strip() for line in lines if line.strip()]
+
+    def _build_auto_variants(
+        self,
+        *,
+        base_prompt: str,
+        role: str,
+        metrics: list[str],
+        auto_count: int,
+    ) -> list[tuple[str, str]]:
+        metrics_text = ", ".join(metrics) if metrics else "핵심 지표"
+        base_prompt = base_prompt.strip() or "사용자 요청에 충실히 답변하라."
+        variants = [
+            ("base", base_prompt),
+            (
+                "role_focus",
+                f"{base_prompt}\n\nRole: {role}. 이 역할에 맞는 톤을 유지하라.",
+            ),
+            (
+                "metric_focus",
+                f"{base_prompt}\n\n성과 지표({metrics_text})에 맞춰 응답 품질을 높여라.",
+            ),
+            (
+                "concise",
+                f"{base_prompt}\n\n핵심만 간결하게 답하고 필요한 경우 bullet로 정리하라.",
+            ),
+            (
+                "assumptions",
+                f"{base_prompt}\n\n불확실하면 가정과 전제를 명시하라.",
+            ),
+        ]
+        if auto_count <= len(variants):
+            return variants[:auto_count]
+        extra = []
+        for index in range(len(variants) + 1, auto_count + 1):
+            extra.append((f"variant_{index}", f"{base_prompt}\n\n추가 후보 {index}."))
+        return variants + extra

evalvault/domain/services/prompt_registry.py

@@ -7,7 +7,13 @@ from datetime import datetime
 from hashlib import sha256
 from typing import Any
 
-from evalvault.domain.entities.prompt import Prompt, PromptSet, PromptSetBundle, PromptSetItem
+from evalvault.domain.entities.prompt import (
+    Prompt,
+    PromptKind,
+    PromptSet,
+    PromptSetBundle,
+    PromptSetItem,
+)
 
 
 @dataclass(frozen=True)
@@ -16,7 +22,7 @@ class PromptInput:
 
     content: str
     name: str
-    kind: str
+    kind: PromptKind
     role: str
     source: str | None = None
     notes: str | None = None
@@ -99,3 +105,35 @@ def build_prompt_summary(bundle: PromptSetBundle) -> dict[str, Any]:
     if ragas_checksums:
         summary["ragas_prompt_checksums"] = ragas_checksums
     return summary
+
+
+def build_prompt_inputs_from_snapshots(
+    snapshots: dict[str, dict[str, Any]] | None,
+) -> list[PromptInput]:
+    if not snapshots:
+        return []
+    prompt_inputs: list[PromptInput] = []
+    for metric_name, entry in snapshots.items():
+        prompt_text = entry.get("prompt") if isinstance(entry, dict) else None
+        if not isinstance(prompt_text, str):
+            continue
+        prompt_text = prompt_text.strip()
+        if not prompt_text:
+            continue
+        source = entry.get("source") if isinstance(entry, dict) else None
+        metadata = {
+            key: value
+            for key, value in entry.items()
+            if key != "prompt" and isinstance(entry, dict)
+        }
+        prompt_inputs.append(
+            PromptInput(
+                content=prompt_text,
+                name=f"ragas.{metric_name}",
+                kind="ragas",
+                role=str(metric_name),
+                source=source if isinstance(source, str) and source else "ragas",
+                metadata=metadata or None,
+            )
+        )
+    return prompt_inputs

evalvault/domain/services/prompt_scoring_service.py

@@ -0,0 +1,286 @@
+from __future__ import annotations
+
+import logging
+from dataclasses import replace
+from typing import Any
+
+from evalvault.domain.entities import (
+    Dataset,
+    EvaluationRun,
+    PromptCandidate,
+    PromptCandidateSampleScore,
+    PromptCandidateScore,
+    TestCase,
+)
+from evalvault.domain.services.evaluator import RagasEvaluator
+from evalvault.ports.outbound.llm_port import GenerationOptions, LLMPort
+
+logger = logging.getLogger(__name__)
+
+_PROMPT_LABELS_KO = {
+    "system": "시스템",
+    "context": "컨텍스트",
+    "question": "질문",
+    "answer": "답변",
+}
+_PROMPT_LABELS_EN = {
+    "system": "System",
+    "context": "Context",
+    "question": "Question",
+    "answer": "Answer",
+}
+
+
+class PromptScoringService:
+    def __init__(self, evaluator: RagasEvaluator, llm: LLMPort) -> None:
+        self._evaluator = evaluator
+        self._llm = llm
+
+    async def score_candidates(
+        self,
+        *,
+        base_run: EvaluationRun,
+        dev_dataset: Dataset,
+        holdout_dataset: Dataset,
+        candidates: list[PromptCandidate],
+        metrics: list[str],
+        weights: dict[str, float],
+        generation_options: GenerationOptions | None = None,
+        selection_policy: str = "best",
+        selection_index: int | None = None,
+        prompt_language: str | None = None,
+    ) -> list[PromptCandidateScore]:
+        if not metrics:
+            raise ValueError("metrics must not be empty")
+        resolved_weights = _resolve_weights(metrics, weights)
+        scoring_dataset = _resolve_scoring_dataset(dev_dataset, holdout_dataset)
+
+        sample_count = _resolve_sample_count(generation_options)
+        resolved_language = _resolve_prompt_language(scoring_dataset, prompt_language)
+
+        scored: list[PromptCandidateScore] = []
+        for candidate in candidates:
+            sample_scores: list[PromptCandidateSampleScore] = []
+            for sample_index in range(sample_count):
+                sample_options = _normalize_generation_options(
+                    generation_options,
+                    sample_index,
+                )
+                generated, responses = await self._generate_candidate_dataset(
+                    dataset=scoring_dataset,
+                    system_prompt=candidate.content,
+                    base_run_id=base_run.run_id,
+                    generation_options=sample_options,
+                    prompt_language=resolved_language,
+                )
+                run = await self._evaluator.evaluate(
+                    dataset=generated,
+                    metrics=metrics,
+                    llm=self._llm,
+                    thresholds=generated.thresholds,
+                    parallel=False,
+                    batch_size=5,
+                )
+                scores = _extract_scores(metrics, run)
+                weighted_score = _weighted_score(scores, resolved_weights)
+                sample_scores.append(
+                    PromptCandidateSampleScore(
+                        sample_index=sample_index,
+                        scores=scores,
+                        weighted_score=weighted_score,
+                        responses=responses,
+                    )
+                )
+
+            selected = _select_sample_score(
+                sample_scores,
+                selection_policy=selection_policy,
+                selection_index=selection_index,
+            )
+            scored.append(
+                PromptCandidateScore(
+                    candidate_id=candidate.candidate_id,
+                    scores=selected.scores,
+                    weighted_score=selected.weighted_score,
+                    sample_scores=sample_scores,
+                    selected_sample_index=selected.sample_index,
+                )
+            )
+        return scored
+
+    async def _generate_candidate_dataset(
+        self,
+        *,
+        dataset: Dataset,
+        system_prompt: str,
+        base_run_id: str,
+        generation_options: GenerationOptions | None,
+        prompt_language: str,
+    ) -> tuple[Dataset, list[dict[str, Any]]]:
+        test_cases: list[TestCase] = []
+        responses: list[dict[str, Any]] = []
+        for test_case in dataset.test_cases:
+            prompt = _build_generation_prompt(
+                system_prompt,
+                test_case,
+                language=prompt_language,
+            )
+            try:
+                answer = await self._llm.agenerate_text(
+                    prompt,
+                    options=generation_options,
+                )
+            except Exception as exc:
+                logger.warning("Prompt candidate generation failed: %s", exc)
+                answer = ""
+            test_cases.append(
+                replace(
+                    test_case,
+                    answer=answer,
+                    metadata={
+                        **(test_case.metadata or {}),
+                        "prompt_candidate": True,
+                    },
+                )
+            )
+            responses.append(
+                {
+                    "test_case_id": test_case.id,
+                    "question": test_case.question,
+                    "answer": answer,
+                    "contexts": list(test_case.contexts or []),
+                    "ground_truth": test_case.ground_truth,
+                }
+            )
+        metadata = dict(dataset.metadata)
+        metadata.setdefault("base_run_id", base_run_id)
+        return (
+            Dataset(
+                name=dataset.name,
+                version=dataset.version,
+                test_cases=test_cases,
+                metadata=metadata,
+                source_file=dataset.source_file,
+                thresholds=dict(dataset.thresholds),
+            ),
+            responses,
+        )
+
+
+def _resolve_prompt_language(dataset: Dataset, prompt_language: str | None) -> str:
+    normalized = _normalize_language_hint(prompt_language)
+    if normalized:
+        return normalized
+    metadata = dataset.metadata if isinstance(dataset.metadata, dict) else {}
+    for key in ("language", "lang", "locale"):
+        normalized = _normalize_language_hint(metadata.get(key))
+        if normalized:
+            return normalized
+    languages = metadata.get("languages")
+    if isinstance(languages, list | tuple | set):
+        for entry in languages:
+            normalized = _normalize_language_hint(entry)
+            if normalized:
+                return normalized
+    return "ko"
+
+
+def _normalize_language_hint(value: Any) -> str | None:
+    if value is None:
+        return None
+    text = str(value).strip().lower()
+    if text in {"ko", "kor", "korean", "ko-kr", "kor-hang", "kr"}:
+        return "ko"
+    if text in {"en", "eng", "english", "en-us", "en-gb"}:
+        return "en"
+    return None
+
+
+def _resolve_sample_count(options: GenerationOptions | None) -> int:
+    if options is None or options.n is None:
+        return 1
+    return max(1, options.n)
+
+
+def _normalize_generation_options(
+    options: GenerationOptions | None,
+    sample_index: int,
+) -> GenerationOptions | None:
+    if options is None:
+        return None
+    seed = options.seed + sample_index if options.seed is not None else None
+    return GenerationOptions(
+        temperature=options.temperature,
+        top_p=options.top_p,
+        max_tokens=options.max_tokens,
+        seed=seed,
+    )
+
+
+def _select_sample_score(
+    sample_scores: list[PromptCandidateSampleScore],
+    *,
+    selection_policy: str,
+    selection_index: int | None,
+) -> PromptCandidateSampleScore:
+    if not sample_scores:
+        raise ValueError("No sample scores available")
+    if selection_policy == "best":
+        return max(sample_scores, key=lambda entry: entry.weighted_score)
+    if selection_policy == "index":
+        if selection_index is None:
+            raise ValueError("selection_index is required for index policy")
+        if selection_index < 0 or selection_index >= len(sample_scores):
+            raise ValueError("selection_index out of range")
+        return sample_scores[selection_index]
+    raise ValueError("Unsupported selection_policy")
+
+
+def _resolve_scoring_dataset(dev_dataset: Dataset, holdout_dataset: Dataset) -> Dataset:
+    if holdout_dataset.test_cases:
+        return holdout_dataset
+    if dev_dataset.test_cases:
+        return dev_dataset
+    raise ValueError("No test cases available for scoring")
+
+
+def _resolve_weights(metrics: list[str], weights: dict[str, float]) -> dict[str, float]:
+    if not weights:
+        base = 1.0 / len(metrics)
+        return dict.fromkeys(metrics, base)
+    resolved = {metric: float(weights.get(metric, 0.0)) for metric in metrics}
+    total = sum(resolved.values())
+    if total <= 0:
+        base = 1.0 / len(metrics)
+        return dict.fromkeys(metrics, base)
+    return {metric: value / total for metric, value in resolved.items()}
+
+
+def _extract_scores(metrics: list[str], run: EvaluationRun) -> dict[str, float]:
+    scores: dict[str, float] = {}
+    for metric in metrics:
+        avg = run.get_avg_score(metric)
+        scores[metric] = float(avg) if avg is not None else 0.0
+    return scores
+
+
+def _weighted_score(scores: dict[str, float], weights: dict[str, float]) -> float:
+    return sum(scores.get(metric, 0.0) * weight for metric, weight in weights.items())
+
+
+def _build_generation_prompt(
+    system_prompt: str,
+    test_case: TestCase,
+    *,
+    language: str,
+) -> str:
+    context_block = (
+        "\n".join(f"- {ctx}" for ctx in test_case.contexts) if test_case.contexts else "-"
+    )
+    labels = _PROMPT_LABELS_EN if language == "en" else _PROMPT_LABELS_KO
+    return (
+        f"[{labels['system']}]\n{system_prompt.strip()}\n\n"
+        f"[{labels['context']}]\n{context_block}\n\n"
+        f"[{labels['question']}]\n{test_case.question.strip()}\n\n"
+        f"[{labels['answer']}]\n"
+    )