evalvault 1.57.1__py3-none-any.whl → 1.59.0__py3-none-any.whl

evalvault/domain/services/pipeline_template_registry.py

@@ -51,136 +51,22 @@ class PipelineTemplateRegistry:
         )
         self._templates[AnalysisIntent.ANALYZE_PATTERNS] = self._create_analyze_patterns_template()
         self._templates[AnalysisIntent.ANALYZE_TRENDS] = self._create_analyze_trends_template()
-        self._templates[AnalysisIntent.BENCHMARK_RETRIEVAL] = (
-            self._create_benchmark_retrieval_template()
-        )
-
-        # 보고서 템플릿
-        self._templates[AnalysisIntent.GENERATE_SUMMARY] = self._create_generate_summary_template()
-        self._templates[AnalysisIntent.GENERATE_DETAILED] = (
-            self._create_generate_detailed_template()
+        self._templates[AnalysisIntent.ANALYZE_STATISTICAL] = (
+            self._create_analyze_statistical_template()
         )
-        self._templates[AnalysisIntent.GENERATE_COMPARISON] = (
-            self._create_generate_comparison_template()
-        )
-
-    def get_template(self, intent: AnalysisIntent) -> AnalysisPipeline | None:
-        """의도에 대한 템플릿 조회.
-
-        Args:
-            intent: 분석 의도
-
-        Returns:
-            파이프라인 템플릿 또는 None
-        """
-        return self._templates.get(intent)
-
-    def list_all(self) -> list[tuple[AnalysisIntent, AnalysisPipeline]]:
-        """모든 템플릿 목록.
-
-        Returns:
-            (의도, 템플릿) 튜플 목록
-        """
-        return list(self._templates.items())
-
-    # =========================================================================
-    # Verification Templates
-    # =========================================================================
-
-    def _create_verify_morpheme_template(self) -> AnalysisPipeline:
-        """형태소 분석 검증 템플릿."""
-        nodes = [
-            AnalysisNode(
-                id="load_data",
-                name="데이터 로드",
-                module="data_loader",
-            ),
-            AnalysisNode(
-                id="morpheme_analysis",
-                name="형태소 분석",
-                module="morpheme_analyzer",
-                depends_on=["load_data"],
-            ),
-            AnalysisNode(
-                id="quality_check",
-                name="품질 검사",
-                module="morpheme_quality_checker",
-                depends_on=["morpheme_analysis"],
-            ),
-            AnalysisNode(
-                id="report",
-                name="검증 보고서",
-                module="verification_report",
-                depends_on=["quality_check"],
-            ),
-        ]
-        return AnalysisPipeline(
-            intent=AnalysisIntent.VERIFY_MORPHEME,
-            nodes=nodes,
+        self._templates[AnalysisIntent.ANALYZE_NLP] = self._create_analyze_nlp_template()
+        self._templates[AnalysisIntent.ANALYZE_CAUSAL] = self._create_analyze_causal_template()
+        self._templates[AnalysisIntent.ANALYZE_NETWORK] = self._create_analyze_network_template()
+        self._templates[AnalysisIntent.ANALYZE_PLAYBOOK] = self._create_analyze_playbook_template()
+        self._templates[AnalysisIntent.DETECT_ANOMALIES] = self._create_detect_anomalies_template()
+        self._templates[AnalysisIntent.FORECAST_PERFORMANCE] = (
+            self._create_forecast_performance_template()
         )
-
-    def _create_verify_embedding_template(self) -> AnalysisPipeline:
-        """임베딩 품질 검증 템플릿."""
-        nodes = [
-            AnalysisNode(
-                id="load_data",
-                name="데이터 로드",
-                module="data_loader",
-            ),
-            AnalysisNode(
-                id="embedding_analysis",
-                name="임베딩 분석",
-                module="embedding_analyzer",
-                depends_on=["load_data"],
-            ),
-            AnalysisNode(
-                id="distribution_check",
-                name="분포 검사",
-                module="embedding_distribution",
-                depends_on=["embedding_analysis"],
-            ),
-            AnalysisNode(
-                id="report",
-                name="검증 보고서",
-                module="verification_report",
-                depends_on=["distribution_check"],
-            ),
-        ]
-        return AnalysisPipeline(
-            intent=AnalysisIntent.VERIFY_EMBEDDING,
-            nodes=nodes,
+        self._templates[AnalysisIntent.GENERATE_HYPOTHESES] = (
+            self._create_generate_hypotheses_template()
         )
-
-    def _create_verify_retrieval_template(self) -> AnalysisPipeline:
-        """검색 품질 검증 템플릿."""
-        nodes = [
-            AnalysisNode(
-                id="load_data",
-                name="데이터 로드",
-                module="data_loader",
-            ),
-            AnalysisNode(
-                id="retrieval_analysis",
-                name="검색 분석",
-                module="retrieval_analyzer",
-                depends_on=["load_data"],
-            ),
-            AnalysisNode(
-                id="quality_check",
-                name="품질 검사",
-                module="retrieval_quality_checker",
-                depends_on=["retrieval_analysis"],
-            ),
-            AnalysisNode(
-                id="report",
-                name="검증 보고서",
-                module="verification_report",
-                depends_on=["quality_check"],
-            ),
-        ]
-        return AnalysisPipeline(
-            intent=AnalysisIntent.VERIFY_RETRIEVAL,
-            nodes=nodes,
+        self._templates[AnalysisIntent.BENCHMARK_RETRIEVAL] = (
+            self._create_benchmark_retrieval_template()
         )
 
     # =========================================================================
@@ -441,6 +327,190 @@ class PipelineTemplateRegistry:
             nodes=nodes,
         )
 
+    def _create_analyze_statistical_template(self) -> AnalysisPipeline:
+        nodes = [
+            AnalysisNode(
+                id="load_data",
+                name="데이터 로드",
+                module="data_loader",
+            ),
+            AnalysisNode(
+                id="statistics",
+                name="통계 분석",
+                module="statistical_analyzer",
+                depends_on=["load_data"],
+            ),
+        ]
+        return AnalysisPipeline(
+            intent=AnalysisIntent.ANALYZE_STATISTICAL,
+            nodes=nodes,
+        )
+
+    def _create_analyze_nlp_template(self) -> AnalysisPipeline:
+        nodes = [
+            AnalysisNode(
+                id="load_data",
+                name="데이터 로드",
+                module="data_loader",
+            ),
+            AnalysisNode(
+                id="nlp_analysis",
+                name="NLP 분석",
+                module="nlp_analyzer",
+                depends_on=["load_data"],
+            ),
+        ]
+        return AnalysisPipeline(
+            intent=AnalysisIntent.ANALYZE_NLP,
+            nodes=nodes,
+        )
+
+    def _create_analyze_causal_template(self) -> AnalysisPipeline:
+        nodes = [
+            AnalysisNode(
+                id="load_data",
+                name="데이터 로드",
+                module="data_loader",
+            ),
+            AnalysisNode(
+                id="statistics",
+                name="통계 분석",
+                module="statistical_analyzer",
+                depends_on=["load_data"],
+            ),
+            AnalysisNode(
+                id="causal_analysis",
+                name="인과 분석",
+                module="causal_analyzer",
+                depends_on=["load_data", "statistics"],
+            ),
+        ]
+        return AnalysisPipeline(
+            intent=AnalysisIntent.ANALYZE_CAUSAL,
+            nodes=nodes,
+        )
+
+    def _create_analyze_network_template(self) -> AnalysisPipeline:
+        nodes = [
+            AnalysisNode(
+                id="load_data",
+                name="데이터 로드",
+                module="data_loader",
+            ),
+            AnalysisNode(
+                id="statistics",
+                name="통계 분석",
+                module="statistical_analyzer",
+                depends_on=["load_data"],
+            ),
+            AnalysisNode(
+                id="network_analysis",
+                name="네트워크 분석",
+                module="network_analyzer",
+                depends_on=["statistics"],
+            ),
+        ]
+        return AnalysisPipeline(
+            intent=AnalysisIntent.ANALYZE_NETWORK,
+            nodes=nodes,
+        )
+
+    def _create_analyze_playbook_template(self) -> AnalysisPipeline:
+        nodes = [
+            AnalysisNode(
+                id="load_data",
+                name="데이터 로드",
+                module="data_loader",
+            ),
+            AnalysisNode(
+                id="diagnostic",
+                name="진단 분석",
+                module="diagnostic_playbook",
+                depends_on=["load_data"],
+            ),
+        ]
+        return AnalysisPipeline(
+            intent=AnalysisIntent.ANALYZE_PLAYBOOK,
+            nodes=nodes,
+        )
+
+    def _create_detect_anomalies_template(self) -> AnalysisPipeline:
+        nodes = [
+            AnalysisNode(
+                id="load_runs",
+                name="실행 기록 로드",
+                module="run_loader",
+            ),
+            AnalysisNode(
+                id="anomaly_detection",
+                name="이상 탐지",
+                module="timeseries_advanced",
+                params={"mode": "anomaly"},
+                depends_on=["load_runs"],
+            ),
+        ]
+        return AnalysisPipeline(
+            intent=AnalysisIntent.DETECT_ANOMALIES,
+            nodes=nodes,
+        )
+
+    def _create_forecast_performance_template(self) -> AnalysisPipeline:
+        nodes = [
+            AnalysisNode(
+                id="load_runs",
+                name="실행 기록 로드",
+                module="run_loader",
+            ),
+            AnalysisNode(
+                id="forecast",
+                name="성능 예측",
+                module="timeseries_advanced",
+                params={"mode": "forecast"},
+                depends_on=["load_runs"],
+            ),
+        ]
+        return AnalysisPipeline(
+            intent=AnalysisIntent.FORECAST_PERFORMANCE,
+            nodes=nodes,
+        )
+
+    def _create_generate_hypotheses_template(self) -> AnalysisPipeline:
+        nodes = [
+            AnalysisNode(
+                id="load_data",
+                name="데이터 로드",
+                module="data_loader",
+            ),
+            AnalysisNode(
+                id="statistics",
+                name="통계 분석",
+                module="statistical_analyzer",
+                depends_on=["load_data"],
+            ),
+            AnalysisNode(
+                id="ragas_eval",
+                name="RAGAS 평가",
+                module="ragas_evaluator",
+                depends_on=["load_data"],
+            ),
+            AnalysisNode(
+                id="low_samples",
+                name="낮은 성능 케이스 추출",
+                module="low_performer_extractor",
+                depends_on=["ragas_eval"],
+            ),
+            AnalysisNode(
+                id="hypothesis",
+                name="가설 생성",
+                module="hypothesis_generator",
+                depends_on=["statistics", "low_samples"],
+            ),
+        ]
+        return AnalysisPipeline(
+            intent=AnalysisIntent.GENERATE_HYPOTHESES,
+            nodes=nodes,
+        )
+
     def _create_benchmark_retrieval_template(self) -> AnalysisPipeline:
         """검색 벤치마크 템플릿."""
         nodes = [

evalvault/domain/services/retriever_context.py

@@ -3,10 +3,16 @@
 from __future__ import annotations
 
 import time
-from collections.abc import Sequence
+from collections.abc import Callable, Sequence
+from datetime import date
 from typing import Any
 
-from evalvault.domain.entities import Dataset
+from evalvault.domain.entities import Dataset, TestCase
+from evalvault.domain.services.document_versioning import (
+    VersionedChunk,
+    parse_contract_date,
+    select_chunks_for_contract_date,
+)
 from evalvault.ports.outbound.korean_nlp_port import RetrieverPort, RetrieverResultProtocol
 
 
@@ -156,3 +162,51 @@ def _compact_values(values: set[str]) -> str | list[str]:
     if len(values) == 1:
         return next(iter(values))
     return sorted(values)
+
+
+def apply_versioned_retriever_to_dataset(
+    *,
+    dataset: Dataset,
+    versioned_chunks: Sequence[VersionedChunk],
+    build_retriever: Callable[[Sequence[str]], RetrieverPort],
+    top_k: int,
+) -> dict[str, dict[str, Any]]:
+    cases_by_contract: dict[date | None, list[TestCase]] = {}
+    for test_case in dataset.test_cases:
+        if _has_contexts(test_case.contexts):
+            continue
+        contract = None
+        if isinstance(test_case.metadata, dict):
+            contract = parse_contract_date(test_case.metadata.get("contract_date"))
+        cases_by_contract.setdefault(contract, []).append(test_case)
+
+    if not cases_by_contract:
+        return {}
+
+    retrieval_metadata: dict[str, dict[str, Any]] = {}
+    chunk_list = list(versioned_chunks)
+
+    for contract, cases in cases_by_contract.items():
+        selected = select_chunks_for_contract_date(chunk_list, contract)
+        documents = [chunk.content for chunk in selected]
+        doc_ids = [chunk.doc_id for chunk in selected]
+
+        retriever = build_retriever(documents)
+        subset = Dataset(
+            name=dataset.name,
+            version=dataset.version,
+            test_cases=cases,
+            metadata=dict(dataset.metadata or {}),
+            source_file=dataset.source_file,
+            thresholds=dict(dataset.thresholds or {}),
+        )
+        retrieval_metadata.update(
+            apply_retriever_to_dataset(
+                dataset=subset,
+                retriever=retriever,
+                top_k=top_k,
+                doc_ids=doc_ids,
+            )
+        )
+
+    return retrieval_metadata

evalvault/domain/services/visual_space_service.py

@@ -591,7 +591,7 @@ def _resolve_result_count_norm(stage_events: list[StageEvent]) -> float | None:
         if isinstance(doc_ids, list):
             counts.append(len(doc_ids))
             top_k = event.attributes.get("top_k")
-            if isinstance(top_k, (int, float)) and top_k > 0:
+            if isinstance(top_k, int | float) and top_k > 0:
                 top_ks.append(int(top_k))
             else:
                 top_ks.append(len(doc_ids))

evalvault/ports/outbound/analysis_port.py

@@ -1,6 +1,6 @@
 """분석 서비스 인터페이스."""
 
-from typing import TYPE_CHECKING, Protocol
+from typing import TYPE_CHECKING, Literal, Protocol
 
 from evalvault.domain.entities import EvaluationRun
 from evalvault.domain.entities.analysis import (
@@ -44,7 +44,7 @@ class AnalysisPort(Protocol):
         run_a: EvaluationRun,
         run_b: EvaluationRun,
         metrics: list[str] | None = None,
-        test_type: str = "t-test",
+        test_type: Literal["t-test", "mann-whitney"] = "t-test",
     ) -> list[ComparisonResult]:
         """두 실행을 통계적으로 비교합니다.
 

evalvault/ports/outbound/improvement_port.py

@@ -28,6 +28,7 @@ class PatternDefinitionProtocol(Protocol):
     pattern_type: str
     component: str
     priority: str
+    detection_rules: Sequence[Any]
     actions: Sequence[ActionDefinitionProtocol]
 
 
@@ -55,6 +56,7 @@ class PatternDetectorPort(Protocol):
         metrics: Sequence[str] | None = None,
     ) -> Mapping[str, list[PatternEvidence]]:
         """Detect problematic patterns for the evaluation run."""
+        ...
 
 
 @runtime_checkable
@@ -71,6 +73,7 @@ class InsightGeneratorPort(Protocol):
 
     def enrich_failure_sample(self, failure: FailureSample) -> FailureSample:
         """Enrich a single failure sample using LLM analysis."""
+        ...
 
     def analyze_batch_failures(
         self,
@@ -80,3 +83,4 @@ class InsightGeneratorPort(Protocol):
         threshold: float,
     ) -> ClaimImprovementProtocol:
         """Produce aggregated insights for multiple failures."""
+        ...