evalvault 1.68.1__tar.gz → 1.69.0__tar.gz

{evalvault-1.68.1 → evalvault-1.69.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evalvault
-Version: 1.68.1
+Version: 1.69.0
 Summary: RAG evaluation system using Ragas with Phoenix/Langfuse tracing
 Project-URL: Homepage, https://github.com/ntts9990/EvalVault
 Project-URL: Documentation, https://github.com/ntts9990/EvalVault#readme

{evalvault-1.68.1 → evalvault-1.69.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "evalvault"
-version = "1.68.1"
+version = "1.69.0"
 description = "RAG evaluation system using Ragas with Phoenix/Langfuse tracing"
 readme = "README.md"
 requires-python = ">=3.12"

{evalvault-1.68.1 → evalvault-1.69.0}/src/evalvault/adapters/outbound/analysis/pipeline_factory.py

@@ -58,6 +58,7 @@ def build_analysis_pipeline_service(
     service.register_module(analysis_modules.DiagnosticPlaybookModule())
     service.register_module(analysis_modules.RootCauseAnalyzerModule())
     service.register_module(analysis_modules.PatternDetectorModule())
+    service.register_module(analysis_modules.MultiTurnAnalyzerModule())
     service.register_module(analysis_modules.TimeSeriesAnalyzerModule())
     service.register_module(analysis_modules.TimeSeriesAdvancedModule())
     service.register_module(analysis_modules.TrendDetectorModule())

{evalvault-1.68.1 → evalvault-1.69.0}/src/evalvault/adapters/outbound/analysis/retrieval_analyzer_module.py

@@ -37,6 +37,8 @@ class RetrievalAnalyzerModule(BaseAnalysisModule):
 
         params = params or {}
         max_cases = int(params.get("max_cases", 150))
+        max_examples = int(params.get("max_examples", 5))
+        max_graphrag_docs = int(params.get("max_graphrag_docs", 5))
 
         context_counts: list[int] = []
         context_token_counts: list[int] = []
@@ -96,6 +98,11 @@ class RetrievalAnalyzerModule(BaseAnalysisModule):
         retrieval_meta = run.retrieval_metadata or {}
         retrieval_times: list[float] = []
         retrieval_scores: list[float] = []
+        graph_nodes: list[int] = []
+        graph_edges: list[int] = []
+        subgraph_sizes: list[int] = []
+        graphrag_cases = 0
+        graphrag_examples: list[dict[str, Any]] = []
         for item in retrieval_meta.values():
             if isinstance(item, dict):
                 if "retrieval_time_ms" in item:
@@ -103,6 +110,20 @@ class RetrievalAnalyzerModule(BaseAnalysisModule):
                 scores = item.get("scores")
                 if isinstance(scores, list) and scores:
                     retrieval_scores.append(safe_mean([float(s) for s in scores]))
+                if "graph_nodes" in item:
+                    graph_nodes.append(int(item["graph_nodes"]))
+                if "graph_edges" in item:
+                    graph_edges.append(int(item["graph_edges"]))
+                if "subgraph_size" in item:
+                    subgraph_sizes.append(int(item["subgraph_size"]))
+                if item.get("retriever") == "graphrag":
+                    graphrag_cases += 1
+                    if len(graphrag_examples) < max_examples:
+                        graphrag_details = item.get("graphrag")
+                        if isinstance(graphrag_details, dict):
+                            graphrag_examples.append(
+                                _trim_graphrag_example(graphrag_details, max_docs=max_graphrag_docs)
+                            )
 
         summary = {
             "total_cases": total_cases,
@@ -122,6 +143,15 @@ class RetrievalAnalyzerModule(BaseAnalysisModule):
             summary["avg_retrieval_time_ms"] = round(safe_mean(retrieval_times), 2)
         if retrieval_scores:
             summary["avg_retrieval_score"] = round(safe_mean(retrieval_scores), 4)
+        if graph_nodes:
+            summary["avg_graph_nodes"] = round(safe_mean(graph_nodes), 2)
+        if graph_edges:
+            summary["avg_graph_edges"] = round(safe_mean(graph_edges), 2)
+        if subgraph_sizes:
+            summary["avg_subgraph_size"] = round(safe_mean(subgraph_sizes), 2)
+        if total_cases:
+            summary["graphrag_case_rate"] = round(graphrag_cases / total_cases, 4)
+        summary["graphrag_case_count"] = graphrag_cases
 
         insights = []
         if summary["avg_contexts"] < 1:
@@ -138,6 +168,38 @@ class RetrievalAnalyzerModule(BaseAnalysisModule):
                 "context_token_counts": context_token_counts[:100],
                 "keyword_overlap_scores": keyword_overlap_scores[:100],
                 "faithfulness_scores": faithfulness_scores[:100],
+                "graph_nodes": graph_nodes[:100],
+                "graph_edges": graph_edges[:100],
+                "subgraph_sizes": subgraph_sizes[:100],
+                "graphrag_examples": graphrag_examples,
             },
             "insights": insights,
         }
+
+
+def _trim_graphrag_example(payload: dict[str, Any], *, max_docs: int) -> dict[str, Any]:
+    docs = payload.get("docs")
+    if isinstance(docs, list):
+        trimmed_docs = []
+        for entry in docs[:max_docs]:
+            if not isinstance(entry, dict):
+                continue
+            trimmed_docs.append(_trim_graphrag_doc(entry))
+        docs = trimmed_docs
+    else:
+        docs = []
+    return {
+        "docs": docs,
+        "max_docs": max_docs,
+    }
+
+
+def _trim_graphrag_doc(entry: dict[str, Any]) -> dict[str, Any]:
+    output: dict[str, Any] = {}
+    for key in ("doc_id", "rank", "score"):
+        if key in entry:
+            output[key] = entry[key]
+    sources = entry.get("sources")
+    if isinstance(sources, dict):
+        output["sources"] = sources
+    return output

{evalvault-1.68.1 → evalvault-1.69.0}/src/evalvault/domain/services/pipeline_template_registry.py

@@ -658,6 +658,12 @@ class PipelineTemplateRegistry:
                 module="statistical_analyzer",
                 depends_on=["load_data"],
             ),
+            AnalysisNode(
+                id="retrieval_analysis",
+                name="검색 분석",
+                module="retrieval_analyzer",
+                depends_on=["load_data"],
+            ),
             AnalysisNode(
                 id="priority_summary",
                 name="우선순위 요약",
@@ -669,7 +675,7 @@ class PipelineTemplateRegistry:
                 name="LLM 요약 보고서",
                 module="llm_report",
                 params={"report_type": "summary"},
-                depends_on=["load_data", "statistics"],
+                depends_on=["load_data", "statistics", "retrieval_analysis"],
             ),
         ]
         return AnalysisPipeline(
@@ -698,6 +704,12 @@ class PipelineTemplateRegistry:
                 module="ragas_evaluator",
                 depends_on=["load_data"],
             ),
+            AnalysisNode(
+                id="retrieval_analysis",
+                name="검색 분석",
+                module="retrieval_analyzer",
+                depends_on=["load_data"],
+            ),
             AnalysisNode(
                 id="low_samples",
                 name="낮은 성능 케이스 추출",
@@ -710,6 +722,12 @@ class PipelineTemplateRegistry:
                 module="diagnostic_playbook",
                 depends_on=["load_data", "ragas_eval"],
             ),
+            AnalysisNode(
+                id="multiturn",
+                name="멀티턴 분석",
+                module="multiturn_analyzer",
+                depends_on=["load_data", "ragas_eval"],
+            ),
             AnalysisNode(
                 id="nlp_analysis",
                 name="NLP 분석",
@@ -767,11 +785,13 @@ class PipelineTemplateRegistry:
                     "load_data",
                     "statistics",
                     "ragas_eval",
+                    "retrieval_analysis",
                     "nlp_analysis",
                     "pattern_detection",
                     "causal_analysis",
                     "root_cause",
                     "priority_summary",
+                    "multiturn",
                     "trend_detection",
                 ],
             ),

{evalvault-1.68.1 → evalvault-1.69.0}/src/evalvault/domain/services/retriever_context.py

@@ -48,6 +48,14 @@ def apply_retriever_to_dataset(
         if scores:
             metadata["scores"] = scores
         metadata.update(_extract_graph_attributes(results))
+        graphrag_details = _build_graphrag_details(
+            results,
+            doc_ids=resolved_doc_ids,
+            max_docs=top_k,
+        )
+        if graphrag_details:
+            metadata["retriever"] = "graphrag"
+            metadata["graphrag"] = graphrag_details
         retrieval_metadata[test_case.id] = metadata
 
     return retrieval_metadata
@@ -164,6 +172,114 @@ def _compact_values(values: set[str]) -> str | list[str]:
     return sorted(values)
 
 
+def _build_graphrag_details(
+    results: Sequence[RetrieverResultProtocol],
+    *,
+    doc_ids: Sequence[str],
+    max_docs: int,
+    max_entities: int = 20,
+    max_relations: int = 20,
+) -> dict[str, Any] | None:
+    details: list[dict[str, Any]] = []
+    for rank, result in enumerate(results, start=1):
+        metadata = getattr(result, "metadata", None)
+        if not isinstance(metadata, dict):
+            continue
+
+        kg_meta = metadata.get("kg") if isinstance(metadata.get("kg"), dict) else None
+        bm25_meta = metadata.get("bm25") if isinstance(metadata.get("bm25"), dict) else None
+        dense_meta = metadata.get("dense") if isinstance(metadata.get("dense"), dict) else None
+        community_id = metadata.get("community_id")
+
+        if not (kg_meta or bm25_meta or dense_meta or community_id is not None):
+            continue
+
+        doc_id = _resolve_doc_id(result, doc_ids, rank)
+        entry: dict[str, Any] = {
+            "doc_id": doc_id,
+            "rank": rank,
+        }
+        score = _extract_score(result)
+        if score is not None:
+            entry["score"] = score
+
+        sources: dict[str, Any] = {}
+        if kg_meta:
+            sources["kg"] = {
+                "entity_score": _coerce_float_or_none(kg_meta.get("entity_score")),
+                "relation_score": _coerce_float_or_none(kg_meta.get("relation_score")),
+                "entities": _limit_strings(kg_meta.get("entities"), max_entities),
+                "relations": _limit_strings(kg_meta.get("relations"), max_relations),
+                "community_id": _coerce_text_or_list(kg_meta.get("community_id")),
+            }
+        if bm25_meta:
+            sources["bm25"] = _build_rank_score(bm25_meta)
+        if dense_meta:
+            sources["dense"] = _build_rank_score(dense_meta)
+        if community_id is not None:
+            sources["community_id"] = _coerce_text_or_list(community_id)
+        if sources:
+            entry["sources"] = sources
+
+        details.append(entry)
+        if len(details) >= max_docs:
+            break
+
+    if not details:
+        return None
+
+    return {
+        "docs": details,
+        "max_docs": max_docs,
+        "max_entities": max_entities,
+        "max_relations": max_relations,
+    }
+
+
+def _build_rank_score(payload: dict[str, Any]) -> dict[str, Any]:
+    out: dict[str, Any] = {}
+    rank = _coerce_int_optional(payload.get("rank"))
+    if rank is not None:
+        out["rank"] = rank
+    score = _coerce_float_or_none(payload.get("score"))
+    if score is not None:
+        out["score"] = score
+    return out
+
+
+def _coerce_float_or_none(value: Any) -> float | None:
+    try:
+        if value is None:
+            return None
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _coerce_int_optional(value: Any) -> int | None:
+    try:
+        if value is None:
+            return None
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _coerce_text_or_list(value: Any) -> str | list[str] | None:
+    if value is None:
+        return None
+    if isinstance(value, (list, tuple, set)):
+        return [str(item) for item in value]
+    return str(value)
+
+
+def _limit_strings(value: Any, limit: int) -> list[str]:
+    if not value:
+        return []
+    items = list(value) if isinstance(value, (list, tuple, set)) else [value]
+    return [str(item) for item in items[:limit]]
+
+
 def apply_versioned_retriever_to_dataset(
     *,
     dataset: Dataset,

{evalvault-1.68.1 → evalvault-1.69.0}/tests/unit/domain/services/test_retriever_context.py

@@ -65,6 +65,9 @@ def test_apply_retriever_populates_contexts_and_metadata() -> None:
     assert metadata["tc-1"]["graph_nodes"] == 1
     assert metadata["tc-1"]["graph_edges"] == 1
     assert metadata["tc-1"]["community_id"] == "c1"
+    assert metadata["tc-1"]["retriever"] == "graphrag"
+    assert metadata["tc-1"]["graphrag"]["docs"][0]["doc_id"] == "doc-b"
+    assert metadata["tc-1"]["graphrag"]["docs"][0]["sources"]["kg"]["entities"] == ["e1"]
 
 
 def test_normalize_retrieval_results_ignores_missing_scores() -> None:

{evalvault-1.68.1 → evalvault-1.69.0}/uv.lock

@@ -1163,7 +1163,7 @@ wheels = [
 
 [[package]]
 name = "evalvault"
-version = "1.68.1"
+version = "1.69.0"
 source = { editable = "." }
 dependencies = [
     { name = "chainlit" },