codebuff-agent 2.1.74__py3-none-any.whl

jupyter_ext/handlers/benchmark_handler.py

@@ -0,0 +1,794 @@
+"""RAG Benchmark Handler — 테스트 케이스 관리, 벤치마크 실행, 결과 조회."""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+import uuid
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+from jupyter_server.base.handlers import APIHandler
+
+from codebuff_core.legacy_paths import resolve_home
+
+logger = logging.getLogger(__name__)
+
+# 벤치마크 데이터 저장 경로
+_BENCHMARK_DIR = resolve_home("benchmark")
+_TEST_CASES_FILE = _BENCHMARK_DIR / "test_cases.json"
+_RESULTS_DIR = _BENCHMARK_DIR / "results"
+
+
+def _ensure_dirs():
+    _BENCHMARK_DIR.mkdir(parents=True, exist_ok=True)
+    _RESULTS_DIR.mkdir(parents=True, exist_ok=True)
+
+
+def _load_test_cases() -> List[Dict[str, Any]]:
+    _ensure_dirs()
+    if _TEST_CASES_FILE.exists():
+        try:
+            return json.loads(_TEST_CASES_FILE.read_text(encoding="utf-8"))
+        except Exception:
+            return []
+    return []
+
+
+def _save_test_cases(cases: List[Dict[str, Any]]):
+    _ensure_dirs()
+    _TEST_CASES_FILE.write_text(
+        json.dumps(cases, ensure_ascii=False, indent=2), encoding="utf-8"
+    )
+
+
+def _load_results() -> List[Dict[str, Any]]:
+    """실행 결과 목록 (요약만)."""
+    _ensure_dirs()
+    results = []
+    for f in sorted(_RESULTS_DIR.glob("*.json"), reverse=True):
+        try:
+            data = json.loads(f.read_text(encoding="utf-8"))
+            results.append(
+                {
+                    "id": f.stem,
+                    "date": data.get("date", ""),
+                    "memo": data.get("memo", ""),
+                    "total_cases": data.get("total_cases", 0),
+                    "summary": data.get("summary", {}),
+                }
+            )
+        except Exception:
+            continue
+    return results
+
+
+def _load_result_detail(result_id: str) -> Optional[Dict[str, Any]]:
+    path = _RESULTS_DIR / f"{result_id}.json"
+    if path.exists():
+        return json.loads(path.read_text(encoding="utf-8"))
+    return None
+
+
+def _save_result(result: Dict[str, Any]) -> str:
+    _ensure_dirs()
+    result_id = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:6]}"
+    path = _RESULTS_DIR / f"{result_id}.json"
+    result["id"] = result_id
+    path.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
+    return result_id
+
+
+# ─── Test Cases CRUD ───
+
+
+class BenchmarkTestCasesHandler(APIHandler):
+    """GET/POST /codebuff/benchmark/test-cases"""
+
+    async def get(self):
+        self.write(json.dumps(_load_test_cases(), ensure_ascii=False))
+
+    async def post(self):
+        body = (
+            json.loads(self.request.body.decode("utf-8")) if self.request.body else {}
+        )
+        action = body.get("action", "save_all")
+
+        if action == "save_all":
+            cases = body.get("test_cases", [])
+            _save_test_cases(cases)
+            self.write({"status": "ok", "count": len(cases)})
+
+        elif action == "add":
+            cases = _load_test_cases()
+            new_case = body.get("test_case", {})
+            if not new_case.get("id"):
+                new_case["id"] = f"tc_{uuid.uuid4().hex[:8]}"
+            cases.append(new_case)
+            _save_test_cases(cases)
+            self.write({"status": "ok", "id": new_case["id"]})
+
+        elif action == "delete":
+            tc_id = body.get("id")
+            cases = [c for c in _load_test_cases() if c.get("id") != tc_id]
+            _save_test_cases(cases)
+            self.write({"status": "ok"})
+
+        elif action == "update":
+            tc_id = body.get("id")
+            updated = body.get("test_case", {})
+            cases = _load_test_cases()
+            for i, c in enumerate(cases):
+                if c.get("id") == tc_id:
+                    cases[i] = {**c, **updated}
+                    break
+            _save_test_cases(cases)
+            self.write({"status": "ok"})
+
+
+# ─── Auto-generate test cases ───
+
+
+class BenchmarkGenerateHandler(APIHandler):
+    """POST /codebuff/benchmark/generate — Qdrant 문서에서 QA 쌍 자동 생성."""
+
+    async def post(self):
+        body = (
+            json.loads(self.request.body.decode("utf-8")) if self.request.body else {}
+        )
+        collection = body.get("collection", "guide_docs")
+        sources = body.get("sources", [])  # 선택된 문서 목록
+        per_source = body.get("per_source", 1)
+
+        try:
+            from .rag_admin_handler import _ensure_manager
+
+            mgr = _ensure_manager()
+            if mgr is None:
+                self.set_status(500)
+                self.write({"error": "RAG 매니저 미초기화"})
+                return
+
+            client = mgr._client
+
+            # 선택된 문서의 chunk 가져오기
+            chunks_by_source: Dict[str, List[Dict]] = {}
+            scroll_offset = None
+            while True:
+                pts, scroll_offset = client.scroll(
+                    collection_name=collection,
+                    limit=100,
+                    with_payload=True,
+                    with_vectors=False,
+                    offset=scroll_offset,
+                )
+                for pt in pts:
+                    p = pt.payload or {}
+                    src = p.get("source", p.get("title", "unknown"))
+                    if sources and src not in sources:
+                        continue
+                    content = p.get("content", "")
+                    if len(content) < 100:
+                        continue
+                    chunks_by_source.setdefault(src, []).append(
+                        {
+                            "source": src,
+                            "title": p.get("title", ""),
+                            "section": p.get("section", ""),
+                            "content": content[:3000],
+                        }
+                    )
+                if scroll_offset is None:
+                    break
+
+            if not chunks_by_source:
+                self.write(
+                    json.dumps(
+                        {"test_cases": [], "message": "해당 문서를 찾을 수 없습니다."}
+                    )
+                )
+                return
+
+            # SSE 헤더
+            self.set_header("Content-Type", "text/event-stream")
+            self.set_header("Cache-Control", "no-cache")
+            self.set_header("Connection", "keep-alive")
+
+            # LLM으로 QA 생성
+            from codebuff_core.llm.service import LLMService
+            from codebuff_core.managers.config_manager import get_config_manager
+
+            cfg = get_config_manager().get_config()
+            llm_config = cfg.get("vllm", {})
+            if not llm_config:
+                self.write(
+                    f"data: {json.dumps({'type': 'error', 'error': 'LLM 설정 없음'})}\n\n"
+                )
+                self.finish()
+                return
+
+            llm = LLMService({"provider": "vllm", "vllm": llm_config})
+            generated: List[Dict] = []
+
+            # 전체 chunk 목록 생성
+            all_tasks: List[Dict] = []
+            for src, chunks in chunks_by_source.items():
+                selected = sorted(
+                    chunks, key=lambda c: len(c["content"]), reverse=True
+                )[:per_source]
+                for chunk in selected:
+                    all_tasks.append(chunk)
+
+            total_tasks = len(all_tasks)
+            self.write(
+                f"data: {json.dumps({'type': 'progress', 'current': 0, 'total': total_tasks, 'source': '병렬 생성 시작'}, ensure_ascii=False)}\n\n"
+            )
+            await self.flush()
+
+            # 병렬 LLM 호출 (최대 5개 동시)
+            _PARALLEL = 5
+            _completed = 0
+
+            async def _generate_one(chunk: Dict) -> Optional[Dict]:
+                prompt = (
+                    "아래 문서 내용을 읽고, 이 문서로만 답할 수 있는 질문과 기대 답변을 만들어주세요.\n\n"
+                    "규칙:\n"
+                    "- 실제 사용자가 할 법한 자연어 질문으로 만드세요.\n"
+                    "- 기대 답변(ground_truth)은 문서 내용을 충실히 반영하여 **구체적이고 상세하게** 작성하세요.\n"
+                    "- 핵심 함수명, 파라미터, 절차, 설정값 등을 포함하세요.\n"
+                    "- must_not_contain에는 문서에 없지만 LLM이 만들어낼 수 있는 그럴듯한 키워드를 넣으세요.\n\n"
+                    "반드시 아래 JSON 형식으로만 응답하세요:\n"
+                    '{"question": "질문", "ground_truth": "상세한 기대 답변", '
+                    '"must_not_contain": ["할루시네이션 가능 키워드"]}\n\n'
+                    f"[문서: {chunk['source']}]\n"
+                    f"[섹션: {chunk['section']}]\n\n"
+                    f"{chunk['content']}"
+                )
+                try:
+                    resp = await llm.generate_response(prompt)
+                    resp = resp.strip()
+                    if resp.startswith("```"):
+                        resp = resp.split("```")[1]
+                        if resp.startswith("json"):
+                            resp = resp[4:]
+                    parsed = json.loads(resp)
+                    return {
+                        "id": f"auto_{uuid.uuid4().hex[:8]}",
+                        "category": collection,
+                        "question": parsed.get("question", ""),
+                        "ground_truth": parsed.get("ground_truth", ""),
+                        "must_not_contain": parsed.get("must_not_contain", []),
+                        "source": chunk["source"],
+                        "difficulty": "medium",
+                        "auto_generated": True,
+                    }
+                except Exception as e:
+                    logger.warning(f"QA generation failed for {chunk['source']}: {e}")
+                    return None
+
+            import asyncio as _aio
+
+            sem = _aio.Semaphore(_PARALLEL)
+
+            async def _gen_with_sem(chunk: Dict):
+                nonlocal _completed
+                async with sem:
+                    r = await _generate_one(chunk)
+                    _completed += 1
+                    if isinstance(r, dict) and r is not None:
+                        generated.append(r)
+                        self.write(
+                            f"data: {json.dumps({'type': 'case', 'test_case': r}, ensure_ascii=False)}\n\n"
+                        )
+                    self.write(
+                        f"data: {json.dumps({'type': 'progress', 'current': _completed, 'total': total_tasks, 'source': chunk['source']}, ensure_ascii=False)}\n\n"
+                    )
+                    await self.flush()
+                    return r
+
+            await _aio.gather(*[_gen_with_sem(c) for c in all_tasks])
+
+            # 완료
+            self.write(
+                f"data: {json.dumps({'type': 'complete', 'total': len(generated)})}\n\n"
+            )
+            await self.flush()
+            self.finish()
+
+        except Exception as e:
+            logger.error(f"Benchmark generate failed: {e}", exc_info=True)
+            self.set_status(500)
+            self.write({"error": str(e)})
+
+
+# ─── Available sources (for UI) ───
+
+
+class BenchmarkSourcesHandler(APIHandler):
+    """GET /codebuff/benchmark/sources?collection=guide_docs"""
+
+    async def get(self):
+        collection = self.get_argument("collection", "guide_docs")
+        try:
+            from .rag_admin_handler import _ensure_manager
+
+            mgr = _ensure_manager()
+            if mgr is None:
+                self.write(json.dumps({"sources": []}))
+                return
+
+            client = mgr._client
+            sources: Dict[str, int] = {}
+            scroll_offset = None
+            while True:
+                pts, scroll_offset = client.scroll(
+                    collection_name=collection,
+                    limit=100,
+                    with_payload=["source", "title"],
+                    with_vectors=False,
+                    offset=scroll_offset,
+                )
+                for pt in pts:
+                    p = pt.payload or {}
+                    src = p.get("source", p.get("title", "unknown"))
+                    sources[src] = sources.get(src, 0) + 1
+                if scroll_offset is None:
+                    break
+
+            result = [{"source": s, "chunks": c} for s, c in sorted(sources.items())]
+            self.write(json.dumps({"sources": result}, ensure_ascii=False))
+
+        except Exception as e:
+            self.write(json.dumps({"sources": [], "error": str(e)}))
+
+
+# ─── Run benchmark ───
+
+
+class BenchmarkRunHandler(APIHandler):
+    """POST /codebuff/benchmark/run — SSE stream으로 벤치마크 실행."""
+
+    async def post(self):
+        body = (
+            json.loads(self.request.body.decode("utf-8")) if self.request.body else {}
+        )
+        test_case_ids = body.get("test_case_ids")  # None이면 전체
+        memo = body.get("memo", "")
+
+        all_cases = _load_test_cases()
+        if test_case_ids:
+            cases = [c for c in all_cases if c.get("id") in test_case_ids]
+        else:
+            cases = all_cases
+
+        if not cases:
+            self.set_status(400)
+            self.write({"error": "테스트 케이스가 없습니다."})
+            return
+
+        # SSE 헤더
+        self.set_header("Content-Type", "text/event-stream")
+        self.set_header("Cache-Control", "no-cache")
+        self.set_header("Connection", "keep-alive")
+
+        from codebuff_core.managers.config_manager import get_config_manager
+
+        cfg = get_config_manager().get_config()
+
+        results: List[Dict] = []
+        total = len(cases)
+        _completed = 0
+        _cancelled = False
+        _PARALLEL = 5
+
+        import asyncio as _aio
+
+        sem = _aio.Semaphore(_PARALLEL)
+
+        async def _run_one(tc):
+            nonlocal _completed, _cancelled
+            if _cancelled:
+                return None
+            async with sem:
+                if _cancelled:
+                    return None
+                try:
+                    r = await self._run_single_case(tc, cfg)
+                except Exception as e:
+                    logger.error(f"Benchmark case {tc['id']} failed: {e}")
+                    r = {"case_id": tc["id"], "error": str(e), "scores": {}}
+                _completed += 1
+                results.append(r)
+                try:
+                    self.write(
+                        f"data: {json.dumps({'type': 'case_result', 'result': r}, ensure_ascii=False, default=str)}\n\n"
+                    )
+                    self.write(
+                        f"data: {json.dumps({'type': 'progress', 'current': _completed, 'total': total})}\n\n"
+                    )
+                    await self.flush()
+                except Exception:
+                    _cancelled = True
+                return r
+
+        try:
+            await _aio.gather(*[_run_one(tc) for tc in cases])
+        except Exception:
+            _cancelled = True
+
+        # 완료된 결과만 저장 (취소 시에도)
+        valid_results = [r for r in results if r is not None]
+        summary = self._calc_summary(valid_results)
+
+        full_result = {
+            "date": datetime.now().isoformat(),
+            "memo": memo + (" (중단됨)" if _cancelled else ""),
+            "total_cases": len(valid_results),
+            "summary": summary,
+            "results": valid_results,
+        }
+        result_id = _save_result(full_result)
+
+        try:
+            self.write(
+                f"data: {json.dumps({'type': 'complete', 'result_id': result_id, 'summary': summary, 'cancelled': _cancelled}, ensure_ascii=False, default=str)}\n\n"
+            )
+            await self.flush()
+        except Exception:
+            pass
+        self.finish()
+
+    async def _run_single_case(
+        self, tc: Dict[str, Any], cfg: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        """단일 테스트 케이스 실행 + 평가."""
+        from codebuff_core.services.rag_chat_agent import RAGChatAgent
+
+        llm_cfg = {"provider": "vllm", "vllm": cfg.get("vllm", {})}
+
+        # knowledge chat과 동일한 프롬프트 사용 — 에이전트가 knowledge 빌더와 centri
+        # `prompts.chat_rag_prompt`(grounding)를 직접 읽는다. `chat_system_prompt` 는
+        # simple 모드 키라 여기에 싣지 않는다 (이슈 #178)
+        # 1. Knowledge Chat으로 질문 실행
+        thread_id = f"benchmark-{uuid.uuid4().hex[:8]}"
+        from langgraph.checkpoint.memory import MemorySaver
+
+        agent = RAGChatAgent(
+            llm_cfg,
+            checkpointer=MemorySaver(),
+        )
+        response_text = ""
+        search_count = 0
+        start_time = time.time()
+
+        async for chunk in agent.stream_response(
+            user_message=tc["question"],
+            thread_id=thread_id,
+        ):
+            if isinstance(chunk, dict):
+                if chunk.get("content"):
+                    response_text += chunk["content"]
+                if chunk.get("status") and "검색 완료" in chunk.get("status", ""):
+                    search_count += 1
+
+        # agent state에서 ToolMessage 추출 (compression 적용 — LLM이 실제로 본 것)
+        rag_results_text = ""
+        _REFLECTION_MARKER = "[검색 결과 분석 지시]"
+        try:
+            from langchain_core.messages import ToolMessage as _TM
+            from codebuff_core.services.rag_chat_agent import (
+                _create_context_compression_middleware_class,
+            )
+
+            if agent._agent is not None:
+                _config = {"configurable": {"thread_id": thread_id}}
+                _state = await agent._agent.aget_state(_config)
+                _raw_msgs = list(_state.values.get("messages", []))
+
+                # compression 적용 (LLM이 실제로 본 compressed 버전)
+                _compressor = _create_context_compression_middleware_class()()
+                _compressed = _compressor.before_model({"messages": _raw_msgs})
+                _msgs = _compressed["messages"] if _compressed else _raw_msgs
+
+                for _msg in _msgs:
+                    if isinstance(_msg, _TM) and _msg.content:
+                        _content = _msg.content
+                        if _REFLECTION_MARKER in _content:
+                            _content = _content[: _content.index(_REFLECTION_MARKER)]
+                        if _content.strip():
+                            rag_results_text += _content[:8000] + "\n"
+                            search_count += 1
+        except Exception as _state_err:
+            logger.warning(f"[Benchmark] {tc['id']}: aget_state failed: {_state_err}")
+
+        # fallback: 검색 결과를 못 가져왔으면 응답 자체를 근거로 사용 (평가 정확도 저하 방지)
+        if not rag_results_text and response_text:
+            rag_results_text = (
+                "[검색 결과를 직접 추출하지 못했습니다. "
+                "아래 응답 내용에서 출처 표기를 참고하여 faithfulness를 평가하세요.]\n"
+                + response_text[:4000]
+            )
+            logger.warning(
+                f"[Benchmark] {tc['id']}: using response as fallback evidence"
+            )
+
+        # search_count fallback: ToolMessage 수로 추정
+        if search_count == 0 and rag_results_text:
+            search_count = rag_results_text.count('"success"')
+
+        elapsed = round(time.time() - start_time, 1)
+        re_search_count = max(search_count - 1, 0)
+
+        # 2. LLM 기반 평가 (검색 결과 포함)
+        scores = await self._evaluate(tc, response_text, rag_results_text, cfg)
+        scores["response_time"] = elapsed
+        scores["search_count"] = search_count
+        scores["re_search_count"] = re_search_count
+
+        # effort bonus: 재검색 시도에 대한 가산 (max +5)
+        effort_bonus = min(re_search_count, 2) * 2.5
+        scores["effort_bonus"] = effort_bonus
+        scores["overall"] = min(round(scores.get("overall", 0) + effort_bonus, 1), 100)
+
+        # Bad test case auto-detection: 검색 5회 이상 + 점수 30 미만 → 데이터 불일치
+        data_mismatch = search_count >= 5 and scores.get("overall", 0) < 30
+        if data_mismatch:
+            logger.info(
+                "[Benchmark] %s: data_mismatch detected "
+                "(search_count=%d, overall=%.1f)",
+                tc["id"],
+                search_count,
+                scores.get("overall", 0),
+            )
+
+        result = {
+            "case_id": tc["id"],
+            "question": tc["question"],
+            "response": response_text,
+            "scores": scores,
+        }
+        if data_mismatch:
+            result["data_mismatch"] = True
+            result["data_mismatch_reason"] = (
+                f"검색 {search_count}회 시도했으나 점수 {scores.get('overall', 0):.1f}/100. "
+                "테스트 케이스의 질문이 현재 RAG 데이터와 맞지 않을 수 있습니다."
+            )
+        return result
+
+    async def _fetch_source_content(self, source: str, collection: str) -> str:
+        """테스트 케이스의 source 문서 원문을 Qdrant에서 직접 가져옴."""
+        try:
+            from .rag_admin_handler import _ensure_manager
+
+            mgr = _ensure_manager()
+            if mgr is None:
+                return ""
+
+            client = mgr._client
+            chunks: List[str] = []
+            scroll_offset = None
+            while True:
+                pts, scroll_offset = client.scroll(
+                    collection_name=collection,
+                    limit=100,
+                    with_payload=True,
+                    with_vectors=False,
+                    offset=scroll_offset,
+                )
+                for pt in pts:
+                    p = pt.payload or {}
+                    src = p.get("source", p.get("title", ""))
+                    # 정확 매칭 또는 부분 매칭 (source 필드에 파일명 포함)
+                    if src == source or source in src or src in source:
+                        content = p.get("content", "")
+                        section = p.get("section", "")
+                        if content:
+                            header = f"[{section}] " if section else ""
+                            chunks.append(f"{header}{content}")
+                if scroll_offset is None:
+                    break
+
+            print(
+                f"[Benchmark] _fetch_source_content: "
+                f"source={source!r} collection={collection!r} "
+                f"chunks={len(chunks)}",
+                flush=True,
+            )
+            return "\n\n---\n\n".join(chunks)
+        except Exception as e:
+            print(
+                f"[Benchmark] _fetch_source_content FAILED: {e}",
+                flush=True,
+            )
+            return ""
+
+    async def _evaluate(
+        self,
+        tc: Dict[str, Any],
+        response: str,
+        rag_results: str,
+        cfg: Dict[str, Any],
+    ) -> Dict[str, Any]:
+        """LLM 기반 평가: Faithfulness + Quality.
+
+        평가 근거로 에이전트 검색 결과 + 원본 문서 내용을 모두 사용.
+        원본 문서를 직접 참조하여 faithfulness 오판을 방지.
+        """
+        from codebuff_core.llm.service import LLMService
+
+        llm_cfg = cfg.get("vllm", {})
+        llm = LLMService({"provider": "vllm", "vllm": llm_cfg})
+
+        # 원본 문서 내용 가져오기 (테스트 케이스의 source 필드 활용)
+        source_content = ""
+        tc_source = tc.get("source", "")
+        tc_collection = tc.get("category", "guide_docs")
+        if tc_source:
+            source_content = await self._fetch_source_content(tc_source, tc_collection)
+
+        # 평가 근거: 원본 문서 + 에이전트 검색 결과 결합
+        print(
+            f"[Benchmark] _evaluate: tc_source={tc_source!r} "
+            f"source_content_len={len(source_content)} "
+            f"rag_results_len={len(rag_results)}",
+            flush=True,
+        )
+        if source_content and rag_results:
+            evidence = (
+                f"[원본 문서 ({tc_source})]\n{source_content[:8000]}\n\n"
+                f"[에이전트 검색 결과]\n{rag_results[:12000]}"
+            )
+        elif source_content:
+            evidence = f"[원본 문서 ({tc_source})]\n{source_content[:12000]}"
+        else:
+            evidence = f"[에이전트 검색 결과]\n{rag_results[:16000]}"
+
+        eval_prompt = (
+            "아래 [질문], [근거 자료], [기대 답변], [실제 응답]을 비교 평가하세요.\n\n"
+            f"[질문]\n{tc['question']}\n\n"
+            f"[근거 자료]\n{evidence}\n\n"
+            f"[기대 답변]\n{tc.get('ground_truth', '없음')}\n\n"
+            f"[실제 응답]\n{response[:3000]}\n\n"
+            "평가 항목 (각 1~10점):\n"
+            "- correctness: 기대 답변과의 일치도\n"
+            "- completeness: 답변 완성도\n"
+            "- faithfulness: 실제 응답이 [근거 자료]에 있는 내용을 바탕으로 답변했는지. "
+            "[근거 자료]에 없는 내용을 만들어냈으면 감점\n"
+            "- conciseness: 핵심만 간결하게 답했는지\n\n"
+            "할루시네이션 판정: 실제 응답의 각 주장이 [근거 자료]에 근거가 있는지 확인하세요. "
+            "[근거 자료]에 근거 없는 주장을 hallucinated_claims에 나열하세요.\n\n"
+            "반드시 아래 JSON만 출력:\n"
+            '{"correctness": N, "completeness": N, "faithfulness": N, '
+            '"conciseness": N, "hallucinated_claims": ["근거 자료에 없는 주장"], '
+            '"comment": "평가 의견"}'
+        )
+
+        import re as _re
+
+        for attempt in range(2):
+            try:
+                resp = await llm.generate_response(eval_prompt)
+                resp = resp.strip()
+                # JSON 추출: 코드블록 또는 raw JSON
+                if resp.startswith("```"):
+                    resp = resp.split("```")[1]
+                    if resp.startswith("json"):
+                        resp = resp[4:]
+                # 텍스트 내 JSON 객체 추출 (fallback)
+                json_match = _re.search(r"\{[^{}]*(?:\[[^\[\]]*\])?[^{}]*\}", resp)
+                if json_match:
+                    resp = json_match.group(0)
+                parsed = json.loads(resp)
+                overall = round(
+                    (
+                        parsed.get("correctness", 5)
+                        + parsed.get("completeness", 5)
+                        + parsed.get("faithfulness", 5)
+                        + parsed.get("conciseness", 5)
+                    )
+                    / 4
+                    * 10,
+                    1,
+                )
+                parsed["overall"] = overall
+
+                # 전항목 0점 감지 → 평가 실패로 간주, 재시도
+                all_zero = all(
+                    parsed.get(k, 0) == 0
+                    for k in (
+                        "correctness",
+                        "completeness",
+                        "faithfulness",
+                        "conciseness",
+                    )
+                )
+                if all_zero and attempt == 0:
+                    logger.warning("Evaluation returned all zeros, retrying")
+                    continue
+                if all_zero:
+                    parsed["parse_error"] = True
+                    parsed["comment"] = (
+                        parsed.get("comment", "") + " [전항목 0점 — 평가 실패 가능성]"
+                    )
+
+                return parsed
+            except Exception as e:
+                if attempt == 0:
+                    logger.warning(f"Evaluation parse failed, retrying: {e}")
+                    continue
+                logger.warning(f"Evaluation failed after retry: {e}")
+                return {
+                    "correctness": 5,
+                    "completeness": 5,
+                    "faithfulness": 5,
+                    "conciseness": 5,
+                    "overall": 50.0,
+                    "hallucinated_claims": [],
+                    "comment": f"평가 파싱 실패 (기본값 적용): {e}",
+                    "parse_error": True,
+                }
+
+    def _calc_summary(self, results: List[Dict]) -> Dict[str, Any]:
+        """전체 결과 요약."""
+        valid = [r for r in results if r.get("scores") and not r["scores"].get("error")]
+        if not valid:
+            return {}
+
+        def avg(key):
+            vals = [r["scores"].get(key, 0) for r in valid if r["scores"].get(key)]
+            return round(sum(vals) / len(vals), 2) if vals else 0
+
+        halluc_counts = []
+        for r in valid:
+            h = r["scores"].get("hallucinated_claims", [])
+            halluc_counts.append(len(h) if isinstance(h, list) else 0)
+
+        data_mismatch_cases = sum(1 for r in results if r.get("data_mismatch"))
+
+        return {
+            "overall": avg("overall"),
+            "correctness": avg("correctness"),
+            "completeness": avg("completeness"),
+            "faithfulness": avg("faithfulness"),
+            "conciseness": avg("conciseness"),
+            "avg_response_time": avg("response_time"),
+            "total_hallucinated_claims": sum(halluc_counts),
+            "cases_with_hallucination": sum(1 for c in halluc_counts if c > 0),
+            "total_evaluated": len(valid),
+            "data_mismatch_cases": data_mismatch_cases,
+        }
+
+
+# ─── Results ───
+
+
+class BenchmarkResultsHandler(APIHandler):
+    """GET/POST /codebuff/benchmark/results — 실행 결과 목록, 상세, 삭제."""
+
+    async def get(self):
+        result_id = self.get_argument("id", None)
+        if result_id:
+            detail = _load_result_detail(result_id)
+            if detail:
+                self.write(json.dumps(detail, ensure_ascii=False, default=str))
+            else:
+                self.set_status(404)
+                self.write({"error": "결과를 찾을 수 없습니다."})
+        else:
+            self.write(json.dumps(_load_results(), ensure_ascii=False, default=str))
+
+    async def post(self):
+        body = (
+            json.loads(self.request.body.decode("utf-8")) if self.request.body else {}
+        )
+        action = body.get("action")
+        if action == "delete":
+            ids = body.get("ids", [])
+            deleted = 0
+            for rid in ids:
+                path = _RESULTS_DIR / f"{rid}.json"
+                if path.exists():
+                    path.unlink()
+                    deleted += 1
+            self.write({"status": "ok", "deleted": deleted})