evalvault 1.68.0__py3-none-any.whl → 1.69.0__py3-none-any.whl

evalvault/adapters/outbound/analysis/pipeline_factory.py

@@ -58,6 +58,7 @@ def build_analysis_pipeline_service(
     service.register_module(analysis_modules.DiagnosticPlaybookModule())
     service.register_module(analysis_modules.RootCauseAnalyzerModule())
     service.register_module(analysis_modules.PatternDetectorModule())
+    service.register_module(analysis_modules.MultiTurnAnalyzerModule())
     service.register_module(analysis_modules.TimeSeriesAnalyzerModule())
     service.register_module(analysis_modules.TimeSeriesAdvancedModule())
     service.register_module(analysis_modules.TrendDetectorModule())

evalvault/adapters/outbound/analysis/retrieval_analyzer_module.py

@@ -37,6 +37,8 @@ class RetrievalAnalyzerModule(BaseAnalysisModule):
 
         params = params or {}
         max_cases = int(params.get("max_cases", 150))
+        max_examples = int(params.get("max_examples", 5))
+        max_graphrag_docs = int(params.get("max_graphrag_docs", 5))
 
         context_counts: list[int] = []
         context_token_counts: list[int] = []
@@ -96,6 +98,11 @@ class RetrievalAnalyzerModule(BaseAnalysisModule):
         retrieval_meta = run.retrieval_metadata or {}
         retrieval_times: list[float] = []
         retrieval_scores: list[float] = []
+        graph_nodes: list[int] = []
+        graph_edges: list[int] = []
+        subgraph_sizes: list[int] = []
+        graphrag_cases = 0
+        graphrag_examples: list[dict[str, Any]] = []
         for item in retrieval_meta.values():
             if isinstance(item, dict):
                 if "retrieval_time_ms" in item:
@@ -103,6 +110,20 @@ class RetrievalAnalyzerModule(BaseAnalysisModule):
                 scores = item.get("scores")
                 if isinstance(scores, list) and scores:
                     retrieval_scores.append(safe_mean([float(s) for s in scores]))
+                if "graph_nodes" in item:
+                    graph_nodes.append(int(item["graph_nodes"]))
+                if "graph_edges" in item:
+                    graph_edges.append(int(item["graph_edges"]))
+                if "subgraph_size" in item:
+                    subgraph_sizes.append(int(item["subgraph_size"]))
+                if item.get("retriever") == "graphrag":
+                    graphrag_cases += 1
+                    if len(graphrag_examples) < max_examples:
+                        graphrag_details = item.get("graphrag")
+                        if isinstance(graphrag_details, dict):
+                            graphrag_examples.append(
+                                _trim_graphrag_example(graphrag_details, max_docs=max_graphrag_docs)
+                            )
 
         summary = {
             "total_cases": total_cases,
@@ -122,6 +143,15 @@ class RetrievalAnalyzerModule(BaseAnalysisModule):
             summary["avg_retrieval_time_ms"] = round(safe_mean(retrieval_times), 2)
         if retrieval_scores:
             summary["avg_retrieval_score"] = round(safe_mean(retrieval_scores), 4)
+        if graph_nodes:
+            summary["avg_graph_nodes"] = round(safe_mean(graph_nodes), 2)
+        if graph_edges:
+            summary["avg_graph_edges"] = round(safe_mean(graph_edges), 2)
+        if subgraph_sizes:
+            summary["avg_subgraph_size"] = round(safe_mean(subgraph_sizes), 2)
+        if total_cases:
+            summary["graphrag_case_rate"] = round(graphrag_cases / total_cases, 4)
+        summary["graphrag_case_count"] = graphrag_cases
 
         insights = []
         if summary["avg_contexts"] < 1:
@@ -138,6 +168,38 @@ class RetrievalAnalyzerModule(BaseAnalysisModule):
                 "context_token_counts": context_token_counts[:100],
                 "keyword_overlap_scores": keyword_overlap_scores[:100],
                 "faithfulness_scores": faithfulness_scores[:100],
+                "graph_nodes": graph_nodes[:100],
+                "graph_edges": graph_edges[:100],
+                "subgraph_sizes": subgraph_sizes[:100],
+                "graphrag_examples": graphrag_examples,
             },
             "insights": insights,
         }
+
+
+def _trim_graphrag_example(payload: dict[str, Any], *, max_docs: int) -> dict[str, Any]:
+    docs = payload.get("docs")
+    if isinstance(docs, list):
+        trimmed_docs = []
+        for entry in docs[:max_docs]:
+            if not isinstance(entry, dict):
+                continue
+            trimmed_docs.append(_trim_graphrag_doc(entry))
+        docs = trimmed_docs
+    else:
+        docs = []
+    return {
+        "docs": docs,
+        "max_docs": max_docs,
+    }
+
+
+def _trim_graphrag_doc(entry: dict[str, Any]) -> dict[str, Any]:
+    output: dict[str, Any] = {}
+    for key in ("doc_id", "rank", "score"):
+        if key in entry:
+            output[key] = entry[key]
+    sources = entry.get("sources")
+    if isinstance(sources, dict):
+        output["sources"] = sources
+    return output

evalvault/domain/services/pipeline_template_registry.py

@@ -658,6 +658,12 @@ class PipelineTemplateRegistry:
                 module="statistical_analyzer",
                 depends_on=["load_data"],
             ),
+            AnalysisNode(
+                id="retrieval_analysis",
+                name="검색 분석",
+                module="retrieval_analyzer",
+                depends_on=["load_data"],
+            ),
             AnalysisNode(
                 id="priority_summary",
                 name="우선순위 요약",
@@ -669,7 +675,7 @@ class PipelineTemplateRegistry:
                 name="LLM 요약 보고서",
                 module="llm_report",
                 params={"report_type": "summary"},
-                depends_on=["load_data", "statistics"],
+                depends_on=["load_data", "statistics", "retrieval_analysis"],
             ),
         ]
         return AnalysisPipeline(
@@ -698,6 +704,12 @@ class PipelineTemplateRegistry:
                 module="ragas_evaluator",
                 depends_on=["load_data"],
             ),
+            AnalysisNode(
+                id="retrieval_analysis",
+                name="검색 분석",
+                module="retrieval_analyzer",
+                depends_on=["load_data"],
+            ),
             AnalysisNode(
                 id="low_samples",
                 name="낮은 성능 케이스 추출",
@@ -710,6 +722,12 @@ class PipelineTemplateRegistry:
                 module="diagnostic_playbook",
                 depends_on=["load_data", "ragas_eval"],
             ),
+            AnalysisNode(
+                id="multiturn",
+                name="멀티턴 분석",
+                module="multiturn_analyzer",
+                depends_on=["load_data", "ragas_eval"],
+            ),
             AnalysisNode(
                 id="nlp_analysis",
                 name="NLP 분석",
@@ -767,11 +785,13 @@ class PipelineTemplateRegistry:
                     "load_data",
                     "statistics",
                     "ragas_eval",
+                    "retrieval_analysis",
                     "nlp_analysis",
                     "pattern_detection",
                     "causal_analysis",
                     "root_cause",
                     "priority_summary",
+                    "multiturn",
                     "trend_detection",
                 ],
             ),

evalvault/domain/services/retriever_context.py

@@ -48,6 +48,14 @@ def apply_retriever_to_dataset(
         if scores:
             metadata["scores"] = scores
         metadata.update(_extract_graph_attributes(results))
+        graphrag_details = _build_graphrag_details(
+            results,
+            doc_ids=resolved_doc_ids,
+            max_docs=top_k,
+        )
+        if graphrag_details:
+            metadata["retriever"] = "graphrag"
+            metadata["graphrag"] = graphrag_details
         retrieval_metadata[test_case.id] = metadata
 
     return retrieval_metadata
@@ -164,6 +172,114 @@ def _compact_values(values: set[str]) -> str | list[str]:
     return sorted(values)
 
 
+def _build_graphrag_details(
+    results: Sequence[RetrieverResultProtocol],
+    *,
+    doc_ids: Sequence[str],
+    max_docs: int,
+    max_entities: int = 20,
+    max_relations: int = 20,
+) -> dict[str, Any] | None:
+    details: list[dict[str, Any]] = []
+    for rank, result in enumerate(results, start=1):
+        metadata = getattr(result, "metadata", None)
+        if not isinstance(metadata, dict):
+            continue
+
+        kg_meta = metadata.get("kg") if isinstance(metadata.get("kg"), dict) else None
+        bm25_meta = metadata.get("bm25") if isinstance(metadata.get("bm25"), dict) else None
+        dense_meta = metadata.get("dense") if isinstance(metadata.get("dense"), dict) else None
+        community_id = metadata.get("community_id")
+
+        if not (kg_meta or bm25_meta or dense_meta or community_id is not None):
+            continue
+
+        doc_id = _resolve_doc_id(result, doc_ids, rank)
+        entry: dict[str, Any] = {
+            "doc_id": doc_id,
+            "rank": rank,
+        }
+        score = _extract_score(result)
+        if score is not None:
+            entry["score"] = score
+
+        sources: dict[str, Any] = {}
+        if kg_meta:
+            sources["kg"] = {
+                "entity_score": _coerce_float_or_none(kg_meta.get("entity_score")),
+                "relation_score": _coerce_float_or_none(kg_meta.get("relation_score")),
+                "entities": _limit_strings(kg_meta.get("entities"), max_entities),
+                "relations": _limit_strings(kg_meta.get("relations"), max_relations),
+                "community_id": _coerce_text_or_list(kg_meta.get("community_id")),
+            }
+        if bm25_meta:
+            sources["bm25"] = _build_rank_score(bm25_meta)
+        if dense_meta:
+            sources["dense"] = _build_rank_score(dense_meta)
+        if community_id is not None:
+            sources["community_id"] = _coerce_text_or_list(community_id)
+        if sources:
+            entry["sources"] = sources
+
+        details.append(entry)
+        if len(details) >= max_docs:
+            break
+
+    if not details:
+        return None
+
+    return {
+        "docs": details,
+        "max_docs": max_docs,
+        "max_entities": max_entities,
+        "max_relations": max_relations,
+    }
+
+
+def _build_rank_score(payload: dict[str, Any]) -> dict[str, Any]:
+    out: dict[str, Any] = {}
+    rank = _coerce_int_optional(payload.get("rank"))
+    if rank is not None:
+        out["rank"] = rank
+    score = _coerce_float_or_none(payload.get("score"))
+    if score is not None:
+        out["score"] = score
+    return out
+
+
+def _coerce_float_or_none(value: Any) -> float | None:
+    try:
+        if value is None:
+            return None
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _coerce_int_optional(value: Any) -> int | None:
+    try:
+        if value is None:
+            return None
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _coerce_text_or_list(value: Any) -> str | list[str] | None:
+    if value is None:
+        return None
+    if isinstance(value, (list, tuple, set)):
+        return [str(item) for item in value]
+    return str(value)
+
+
+def _limit_strings(value: Any, limit: int) -> list[str]:
+    if not value:
+        return []
+    items = list(value) if isinstance(value, (list, tuple, set)) else [value]
+    return [str(item) for item in items[:limit]]
+
+
 def apply_versioned_retriever_to_dataset(
     *,
     dataset: Dataset,

{evalvault-1.68.0.dist-info → evalvault-1.69.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evalvault
-Version: 1.68.0
+Version: 1.69.0
 Summary: RAG evaluation system using Ragas with Phoenix/Langfuse tracing
 Project-URL: Homepage, https://github.com/ntts9990/EvalVault
 Project-URL: Documentation, https://github.com/ntts9990/EvalVault#readme

{evalvault-1.68.0.dist-info → evalvault-1.69.0.dist-info}/RECORD

@@ -90,11 +90,11 @@ evalvault/adapters/outbound/analysis/network_analyzer_module.py,sha256=ITUVnt_CI
 evalvault/adapters/outbound/analysis/nlp_adapter.py,sha256=aLtF_fns-7IEtitwON2EYS_lweq_IdldFsRm47alN0Q,29561
 evalvault/adapters/outbound/analysis/nlp_analyzer_module.py,sha256=kVuG9pVMQO6OYY5zxj_w9nNQZ1-qIO0y6XcXo6lG-n0,8221
 evalvault/adapters/outbound/analysis/pattern_detector_module.py,sha256=SyCDO_VS-r-tjGh8WrW-t1GCSC9ouxirdVk4NizFPXo,1882
-evalvault/adapters/outbound/analysis/pipeline_factory.py,sha256=XvcCbKCN_otv1pGUzk0oE76RV19yFga8r6RngBvgEFo,3691
+evalvault/adapters/outbound/analysis/pipeline_factory.py,sha256=Yk-VPagdAZXbbD08pCSOleg-URuVAzJks4oGl61mKAs,3763
 evalvault/adapters/outbound/analysis/pipeline_helpers.py,sha256=8E8IrYI5JvRrpnjxe0DS7srbPzB0XAxxXhLLYgfwsgU,5756
 evalvault/adapters/outbound/analysis/priority_summary_module.py,sha256=o8Y0rfHjYYE9WNTwKtpJulwfvLA3MNMhYjdSg15Vacc,10802
 evalvault/adapters/outbound/analysis/ragas_evaluator_module.py,sha256=Cd-spGn56zMcqOdoTLUHTYVOFqHqR17tPFyJs7rmnbw,7659
-evalvault/adapters/outbound/analysis/retrieval_analyzer_module.py,sha256=D24GTaKabHacSBI-UqCd_jy61hnne8-QG1p4rqW1Bzk,5748
+evalvault/adapters/outbound/analysis/retrieval_analyzer_module.py,sha256=STRHWapVAEz0YbSxR3NzT6zV7wfwlPxjKZunuWpfTmE,8340
 evalvault/adapters/outbound/analysis/retrieval_benchmark_module.py,sha256=_duIBlYhAsFygEpC7DuwoAqfTbVG2xgp70JjW1LJAGE,9312
 evalvault/adapters/outbound/analysis/retrieval_quality_checker_module.py,sha256=K1IJn4bvvz-BfqQmhd5Ik9oATjq_-G7V1AZSW8zKtSE,3121
 evalvault/adapters/outbound/analysis/root_cause_analyzer_module.py,sha256=UagHWb2d1vD7aCH0vLl3tSJx86gkkxNarrF-rwtEBhU,2811
@@ -280,7 +280,7 @@ evalvault/domain/services/memory_based_analysis.py,sha256=oh2irCy3le7fWiTtL31SME
 evalvault/domain/services/method_runner.py,sha256=pABqKZeaALpWZYDfzAbd-VOZt2djQggRNIPuuPQeUSw,3571
 evalvault/domain/services/ops_snapshot_service.py,sha256=1CqJN2p3tM6SgzLCZKcVEM213fd1cDGexTRPG_3e59w,5138
 evalvault/domain/services/pipeline_orchestrator.py,sha256=yriVlEVZYDtt0Vwt4Ae6xyW1H6Dj4Hxdn8XQSvQNSoQ,19436
-evalvault/domain/services/pipeline_template_registry.py,sha256=aWqXLQ24grpSZo9M4tZLRo1ysD10c6hUpW3JupZH9e0,28083
+evalvault/domain/services/pipeline_template_registry.py,sha256=k5Ce1BC3NgcYqCLiUZpXsl_6WwDHOXONoYDH7KzX2L4,28809
 evalvault/domain/services/prompt_candidate_service.py,sha256=Ibyb5EaWK28Ju2HnTqHHGOoiA9Q-VwY3hjxVODALwGY,3997
 evalvault/domain/services/prompt_manifest.py,sha256=5s5Kd6-_Dn-xrjjlU99CVo6njsPhvE50H5m_85U-H6U,5612
 evalvault/domain/services/prompt_registry.py,sha256=QyL4yIcKT93uv6L0-Q_iaNXno8QnsC19YcGekuSRMtE,5247
@@ -290,7 +290,7 @@ evalvault/domain/services/prompt_suggestion_reporter.py,sha256=Fc6sCPebUMk8SZVpj
 evalvault/domain/services/ragas_prompt_overrides.py,sha256=4BecYE2KrreUBbIM3ssP9WzHcK_wRc8jW7CE_k58QOU,1412
 evalvault/domain/services/regression_gate_service.py,sha256=qBMODgpizmEzqEL8_JX-FYSVyARiroMW7MFVzlz7gjc,6579
 evalvault/domain/services/retrieval_metrics.py,sha256=dtrQPLMrXSyWLcgF8EGcLNFwzwA59WDzEh41JRToHAY,2980
-evalvault/domain/services/retriever_context.py,sha256=ySQ-GuadiggS0LVAib4AxA_0JpasYz4S9hbjau0eyIA,6482
+evalvault/domain/services/retriever_context.py,sha256=TeJ9UgT4l3lXxOXcYMz_7PdVMlV7JsW2ewTXdv9dI2M,10185
 evalvault/domain/services/run_comparison_service.py,sha256=_NScltCRcY3zrvdyYDiPmssTxCDv1GyjCLdP3uAxJts,5631
 evalvault/domain/services/satisfaction_calibration_service.py,sha256=H7Z8opOyPHRO5qVIw-XDsNhIwdCteAS9_a3BTlfIqHg,11906
 evalvault/domain/services/stage_event_builder.py,sha256=FAT34Wmylvd2Yz5rDlhaTh1lqSCDhGApCXMi7Hjkib0,9748
@@ -339,8 +339,8 @@ evalvault/reports/__init__.py,sha256=Bb1X4871msAN8I6PM6nKGED3psPwZt88hXZBAOdH06Y
 evalvault/reports/release_notes.py,sha256=pZj0PBFT-4F_Ty-Kv5P69BuoOnmTCn4kznDcORFJd0w,4011
 evalvault/scripts/__init__.py,sha256=NwEeIFQbkX4ml2R_PhtIoNtArDSX_suuoymgG_7Kwso,89
 evalvault/scripts/regression_runner.py,sha256=SxZori5BZ8jVQ057Mf5V5FPgIVDccrV5oRONmnhuk8w,8438
-evalvault-1.68.0.dist-info/METADATA,sha256=bEWK-9BGROeWrWf3kNoGytr-GbAa2gzLCDZ1PwWBzEM,26159
-evalvault-1.68.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-evalvault-1.68.0.dist-info/entry_points.txt,sha256=Oj9Xc5gYcyUYYNmQfWI8NYGw7nN-3M-h2ipHIMlVn6o,65
-evalvault-1.68.0.dist-info/licenses/LICENSE.md,sha256=3RNWY4jjtrQ_yYa-D-7I3XO12Ti7YzxsLV_dpykujvo,11358
-evalvault-1.68.0.dist-info/RECORD,,
+evalvault-1.69.0.dist-info/METADATA,sha256=dmvwwIQIH_eIRT1ZUxhEsoQbTdoAyXvxiSRuU_zxysg,26159
+evalvault-1.69.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+evalvault-1.69.0.dist-info/entry_points.txt,sha256=Oj9Xc5gYcyUYYNmQfWI8NYGw7nN-3M-h2ipHIMlVn6o,65
+evalvault-1.69.0.dist-info/licenses/LICENSE.md,sha256=3RNWY4jjtrQ_yYa-D-7I3XO12Ti7YzxsLV_dpykujvo,11358
+evalvault-1.69.0.dist-info/RECORD,,