evalvault 1.73.2__py3-none-any.whl → 1.74.0__py3-none-any.whl

evalvault/adapters/inbound/api/adapter.py

@@ -21,6 +21,7 @@ from evalvault.adapters.outbound.analysis import (
 )
 from evalvault.adapters.outbound.cache import MemoryCacheAdapter
 from evalvault.adapters.outbound.judge_calibration_reporter import JudgeCalibrationReporter
+from evalvault.adapters.outbound.ops.report_renderer import render_json, render_markdown
 from evalvault.adapters.outbound.report import MarkdownReportAdapter
 from evalvault.config.phoenix_support import PhoenixExperimentResolver
 from evalvault.config.settings import Settings
@@ -43,6 +44,7 @@ from evalvault.domain.services.analysis_service import AnalysisService
 from evalvault.domain.services.cluster_map_builder import build_cluster_map
 from evalvault.domain.services.debug_report_service import DebugReportService
 from evalvault.domain.services.judge_calibration_service import JudgeCalibrationService
+from evalvault.domain.services.ops_report_service import OpsReportService
 from evalvault.domain.services.prompt_registry import (
     PromptInput,
     build_prompt_bundle,
@@ -1329,6 +1331,42 @@ class WebUIAdapter:
             stage_storage=stage_storage,
         )
 
+    def generate_ops_report(
+        self,
+        run_id: str,
+        *,
+        output_format: str,
+        save: bool,
+    ) -> dict[str, Any] | str:
+        if self._storage is None:
+            raise RuntimeError("Storage not configured")
+        if not hasattr(self._storage, "list_stage_events"):
+            raise RuntimeError("Stage storage not configured")
+
+        service = OpsReportService()
+        stage_storage = cast(StageStoragePort, self._storage)
+        report = service.build_report(
+            run_id,
+            storage=self._storage,
+            stage_storage=stage_storage,
+        )
+
+        content = render_markdown(report) if output_format == "markdown" else render_json(report)
+
+        if save:
+            self._storage.save_ops_report(
+                report_id=None,
+                run_id=run_id,
+                report_type="ops_report",
+                format=output_format,
+                content=content,
+                metadata={"source": "api"},
+            )
+
+        if output_format == "markdown":
+            return content
+        return report.to_dict()
+
     def delete_run(self, run_id: str) -> bool:
         """평가 삭제.
 

evalvault/adapters/inbound/api/routers/chat.py

@@ -44,10 +44,62 @@ class ChatRequest(BaseModel):
     history: list[ChatMessage] | None = None
 
 
+class AiChatMessage(BaseModel):
+    role: str
+    content: str | None = None
+    parts: list[dict[str, Any]] | None = None
+
+
+class AiChatRequest(BaseModel):
+    messages: list[AiChatMessage] = Field(default_factory=list)
+    run_id: str | None = None
+    category: str | None = None
+
+
 def _extract_run_ids(text: str) -> list[str]:
     return re.findall(r"run_[A-Za-z0-9_-]+", text)
 
 
+def _ollama_chat_options(model_name: str) -> dict[str, Any] | None:
+    lower = model_name.lower()
+    if lower.startswith("qwen3"):
+        return {
+            "temperature": 0.6,
+            "top_p": 0.95,
+            "top_k": 20,
+            "repeat_penalty": 1,
+            "stop": ["<|im_start|>", "<|im_end|>"],
+        }
+    return None
+
+
+def _is_verb_only(text: str) -> bool:
+    if not text:
+        return False
+    compact = re.sub(r"\s+", "", text.strip())
+    if not compact:
+        return False
+    tokens = re.findall(r"[A-Za-z0-9가-힣]+", compact)
+    if len(tokens) > 2:
+        return False
+    verb_markers = ["해줘", "해주세요", "해봐", "해봐요", "해줘요", "해줘라"]
+    verb_stems = ["설명", "요약", "분석", "비교", "개선", "정리", "추천", "진단", "해석", "검증"]
+    if any(compact.endswith(marker) for marker in verb_markers):
+        return any(stem in compact for stem in verb_stems)
+    return compact in verb_stems
+
+
+def _with_context(user_text: str, run_id: str | None, category: str | None) -> str:
+    parts = []
+    if run_id:
+        parts.append(f"선택된 run_id: {run_id}")
+    if category:
+        parts.append(f"질문 분류: {category}")
+    if not parts:
+        return user_text
+    return "\n".join(parts) + f"\n사용자 요청: {user_text}"
+
+
 def _format_tool_result(result: Any) -> str:
     if isinstance(result, dict):
         if "result" in result:
@@ -283,15 +335,29 @@ async def warm_rag_index() -> None:
         logger.warning("RAG preload failed: %s", exc)
 
 
-async def _direct_chat_answer(user_text: str) -> str | None:
+async def _direct_chat_answer(
+    user_text: str, run_id: str | None = None, category: str | None = None
+) -> str | None:
+    user_text = _with_context(user_text, run_id, category)
+    model_name = os.getenv("OLLAMA_CHAT_MODEL", "qwen3:14b")
+    options = _ollama_chat_options(model_name)
     payload = {
-        "model": os.getenv("OLLAMA_CHAT_MODEL", "gpt-oss-safeguard:20b"),
+        "model": model_name,
         "messages": [
-            {"role": "system", "content": "You are a helpful assistant for EvalVault."},
+            {
+                "role": "system",
+                "content": (
+                    "You are a helpful assistant for EvalVault. "
+                    "Interpret verb-only requests as questions about the selected run/category. "
+                    "If essential details are missing, ask a concise follow-up question in Korean."
+                ),
+            },
             {"role": "user", "content": user_text},
         ],
         "stream": False,
     }
+    if options:
+        payload["options"] = options
 
     async with httpx.AsyncClient(timeout=30) as client:
         response = await client.post(
@@ -318,7 +384,9 @@ def _simple_retrieve(texts: list[str], query: str, top_k: int) -> list[str]:
     return [text for _, text in scored[:top_k]]
 
 
-async def _rag_answer(user_text: str) -> str | None:
+async def _rag_answer(
+    user_text: str, run_id: str | None = None, category: str | None = None
+) -> str | None:
     retriever, _ = await _get_rag_retriever()
     contexts: list[str] = []
 
@@ -340,22 +408,28 @@ async def _rag_answer(user_text: str) -> str | None:
 
     prompt = (
         "다음은 EvalVault 코드/문서에서 검색된 컨텍스트입니다.\n"
-        "컨텍스트만 근거로 사용해 한국어로 답하세요.\n\n"
+        "컨텍스트만 근거로 사용해 한국어로 답하세요.\n"
+        "질문이 동사만 있는 경우에도 선택된 run_id/분류를 기준으로 해석하세요.\n"
+        "정보가 부족하면 먼저 필요한 정보를 질문하세요.\n\n"
         "[컨텍스트]\n"
         + "\n\n---\n\n".join(contexts[:3])
         + "\n\n[질문]\n"
-        + user_text
+        + _with_context(user_text, run_id, category)
         + "\n\n[답변]"
     )
 
+    model_name = os.getenv("OLLAMA_CHAT_MODEL", "qwen3:14b")
+    options = _ollama_chat_options(model_name)
     payload = {
-        "model": os.getenv("OLLAMA_CHAT_MODEL", "gpt-oss-safeguard:20b"),
+        "model": model_name,
         "messages": [
             {"role": "system", "content": "You are a helpful assistant for EvalVault."},
             {"role": "user", "content": prompt},
         ],
         "stream": False,
     }
+    if options:
+        payload["options"] = options
 
     async with httpx.AsyncClient(timeout=60) as client:
         response = await client.post(
@@ -388,7 +462,9 @@ async def _call_mcp_tool(tool_name: str, tool_args: dict[str, Any]) -> Any:
     return data
 
 
-async def _resolve_tool_with_llm(user_text: str) -> dict[str, Any] | None:
+async def _resolve_tool_with_llm(
+    user_text: str, run_id: str | None = None, category: str | None = None
+) -> dict[str, Any] | None:
     ollama_url = os.getenv("OLLAMA_BASE_URL", "http://localhost:11434")
     router_model = os.getenv("OLLAMA_ROUTER_MODEL", "gemma3:1b")
 
@@ -398,6 +474,8 @@ async def _resolve_tool_with_llm(user_text: str) -> dict[str, Any] | None:
         "Action must be one of: tool, rag, direct."
         "Tools: list_runs, get_run_summary, run_evaluation, analyze_compare, get_artifacts."
         "Rules:"
+        "- Assume verb-only requests refer to the selected run_id/category when provided."
+        "- If essential info is missing (e.g., run_id), return action direct with a follow-up question."
         "- If user asks about datasets, prefer tool list_datasets."
         "- If question is about EvalVault docs/usage, prefer rag."
         "- If greeting or general chat, use direct."
@@ -413,7 +491,7 @@ async def _resolve_tool_with_llm(user_text: str) -> dict[str, Any] | None:
         "model": router_model,
         "messages": [
             {"role": "system", "content": system_prompt},
-            {"role": "user", "content": user_text},
+            {"role": "user", "content": _with_context(user_text, run_id, category)},
         ],
         "stream": False,
     }
@@ -479,6 +557,99 @@ def _chunk_text(text: str, size: int = 42) -> list[str]:
     return [text[i : i + size] for i in range(0, len(text), size)]
 
 
+def _extract_text_from_parts(parts: list[dict[str, Any]] | None) -> str | None:
+    if not parts:
+        return None
+    chunks: list[str] = []
+    for part in parts:
+        if not isinstance(part, dict):
+            continue
+        if part.get("type") == "text":
+            text = part.get("text")
+            if isinstance(text, str) and text:
+                chunks.append(text)
+    if not chunks:
+        return None
+    content = "".join(chunks).strip()
+    return content or None
+
+
+def _extract_last_user_message(messages: list[AiChatMessage]) -> str | None:
+    for message in reversed(messages):
+        if message.role != "user":
+            continue
+        if message.content and message.content.strip():
+            return message.content.strip()
+        content = _extract_text_from_parts(message.parts)
+        if content:
+            return content
+    return None
+
+
+def _ai_sse_event(payload: dict[str, Any]) -> str:
+    return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
+
+
+def _ai_sse_done() -> str:
+    return "data: [DONE]\n\n"
+
+
+def _ai_sse_headers() -> dict[str, str]:
+    return {
+        "Cache-Control": "no-cache",
+        "Connection": "keep-alive",
+        "x-vercel-ai-ui-message-stream": "v1",
+    }
+
+
+async def _ai_chat_stream(
+    user_text: str, run_id: str | None = None, category: str | None = None
+) -> AsyncGenerator[str, None]:
+    message_id = f"msg_{int(time.time() * 1000)}"
+    text_id = f"text_{message_id}"
+    yield _ai_sse_event({"type": "start", "messageId": message_id})
+    yield _ai_sse_event({"type": "text-start", "id": text_id})
+
+    async for item in _chat_stream(user_text, run_id=run_id, category=category):
+        raw = item.strip()
+        if not raw:
+            continue
+        try:
+            payload = json.loads(raw)
+        except Exception:
+            continue
+
+        event_type = payload.get("type")
+        if event_type == "delta":
+            content = payload.get("content")
+            if isinstance(content, str) and content:
+                yield _ai_sse_event({"type": "text-delta", "id": text_id, "delta": content})
+            continue
+        if event_type == "status":
+            message = payload.get("message")
+            if isinstance(message, str) and message:
+                yield _ai_sse_event(
+                    {"type": "data-status", "data": {"message": message}, "transient": True}
+                )
+            continue
+        if event_type == "error":
+            message = payload.get("message")
+            if not isinstance(message, str) or not message:
+                message = "채팅 요청에 실패했습니다."
+            yield _ai_sse_event({"type": "error", "errorText": message})
+            yield _ai_sse_event({"type": "finish"})
+            yield _ai_sse_done()
+            return
+        if event_type == "final":
+            yield _ai_sse_event({"type": "text-end", "id": text_id})
+            yield _ai_sse_event({"type": "finish"})
+            yield _ai_sse_done()
+            return
+
+    yield _ai_sse_event({"type": "finish"})
+    yield _ai_sse_done()
+
+
 def _event(payload: dict[str, Any]) -> str:
     return json.dumps(payload, ensure_ascii=False) + "\n"
 
@@ -490,15 +661,34 @@ async def _emit_answer(answer: str) -> AsyncGenerator[str, None]:
     yield _event({"type": "final", "content": answer})
 
 
-async def _chat_stream(user_text: str) -> AsyncGenerator[str, None]:
+async def _chat_stream(
+    user_text: str, run_id: str | None = None, category: str | None = None
+) -> AsyncGenerator[str, None]:
     started_at = time.perf_counter()
-    if len(user_text) <= 4:
-        yield _event({"type": "final", "content": "안녕하세요! EvalVault 관련 질문을 해주세요."})
+    if category in {"result_interpretation", "improvement_direction"} and not run_id:
+        yield _event(
+            {
+                "type": "final",
+                "content": "선택한 분류는 run_id가 필요합니다. run_id를 선택한 뒤 다시 질문해주세요.",
+            }
+        )
         return
 
-    if len(user_text) <= 6:
+    if len(user_text) <= 4:
+        if run_id or category:
+            user_text = f"{user_text}"
+        else:
+            yield _event(
+                {
+                    "type": "final",
+                    "content": "무엇을 설명할까요? run_id와 질문 분류를 선택한 뒤 다시 요청해주세요.",
+                }
+            )
+            return
+
+    if len(user_text) <= 6 and not _is_verb_only(user_text):
         yield _event({"type": "status", "message": "짧은 질문 처리 중..."})
-        answer = await _direct_chat_answer(user_text)
+        answer = await _direct_chat_answer(user_text, run_id=run_id, category=category)
         if answer:
             async for item in _emit_answer(answer):
                 yield item
@@ -506,9 +696,45 @@ async def _chat_stream(user_text: str) -> AsyncGenerator[str, None]:
             yield _event({"type": "final", "content": "답변을 생성하지 못했습니다."})
         return
 
+    if (
+        _is_verb_only(user_text)
+        and category in {"result_interpretation", "improvement_direction"}
+        and run_id
+    ):
+        yield _event({"type": "status", "message": "선택한 run 요약 중..."})
+        try:
+            result = await asyncio.wait_for(
+                _call_mcp_tool("get_run_summary", {"run_id": run_id}), timeout=12
+            )
+        except TimeoutError:
+            yield _event(
+                {
+                    "type": "error",
+                    "message": "run 요약 응답이 지연됩니다. 잠시 후 다시 시도해주세요.",
+                }
+            )
+            return
+        except Exception as exc:
+            yield _event({"type": "error", "message": f"run 요약 실패: {exc}"})
+            return
+
+        payload = _extract_json_content(result)
+        if isinstance(payload, dict):
+            summary = _summarize_result("get_run_summary", payload)
+            if category == "improvement_direction":
+                summary += "\n\n개선 방향을 구체화하려면 목표 메트릭이나 기준을 알려주세요."
+            else:
+                summary += "\n\n특정 메트릭/케이스가 있으면 알려주세요."
+            async for item in _emit_answer(summary):
+                yield item
+            return
+
     yield _event({"type": "status", "message": "요청 분류 중..."})
     try:
-        router = await asyncio.wait_for(_resolve_tool_with_llm(user_text), timeout=20)
+        router = await asyncio.wait_for(
+            _resolve_tool_with_llm(user_text, run_id=run_id, category=category),
+            timeout=30,
+        )
     except TimeoutError:
         router = None
     except Exception:
@@ -520,7 +746,9 @@ async def _chat_stream(user_text: str) -> AsyncGenerator[str, None]:
     if router is None:
         yield _event({"type": "status", "message": "문서 검색 중..."})
         try:
-            rag_answer = await asyncio.wait_for(_rag_answer(user_text), timeout=30)
+            rag_answer = await asyncio.wait_for(
+                _rag_answer(user_text, run_id=run_id, category=category), timeout=90
+            )
         except TimeoutError:
             yield _event({"type": "error", "message": "문서 검색이 지연됩니다. 다시 시도해주세요."})
             return
@@ -528,7 +756,7 @@ async def _chat_stream(user_text: str) -> AsyncGenerator[str, None]:
             async for item in _emit_answer(rag_answer):
                 yield item
             return
-        answer = await _direct_chat_answer(user_text)
+        answer = await _direct_chat_answer(user_text, run_id=run_id, category=category)
         if answer:
             async for item in _emit_answer(answer):
                 yield item
@@ -541,7 +769,7 @@ async def _chat_stream(user_text: str) -> AsyncGenerator[str, None]:
     tool_args = router.get("arguments", {})
 
     if action == "direct":
-        answer = await _direct_chat_answer(user_text)
+        answer = await _direct_chat_answer(user_text, run_id=run_id, category=category)
         if answer:
             async for item in _emit_answer(answer):
                 yield item
@@ -552,7 +780,9 @@ async def _chat_stream(user_text: str) -> AsyncGenerator[str, None]:
     if action == "rag":
         yield _event({"type": "status", "message": "문서 검색 중..."})
         try:
-            rag_answer = await asyncio.wait_for(_rag_answer(user_text), timeout=30)
+            rag_answer = await asyncio.wait_for(
+                _rag_answer(user_text, run_id=run_id, category=category), timeout=90
+            )
         except TimeoutError:
             yield _event({"type": "error", "message": "문서 검색이 지연됩니다. 다시 시도해주세요."})
             return
@@ -571,9 +801,31 @@ async def _chat_stream(user_text: str) -> AsyncGenerator[str, None]:
         yield _event({"type": "final", "content": "도구 이름을 찾지 못했습니다."})
         return
 
+    if tool_name == "get_run_summary" and not (tool_args.get("run_id") or run_id):
+        yield _event({"type": "final", "content": "run_id를 선택하거나 입력해주세요."})
+        return
+    if tool_name == "get_artifacts" and not (tool_args.get("run_id") or run_id):
+        yield _event({"type": "final", "content": "아티팩트 조회를 위해 run_id가 필요합니다."})
+        return
+    if tool_name == "analyze_compare" and (
+        not tool_args.get("run_id_a") or not tool_args.get("run_id_b")
+    ):
+        yield _event(
+            {
+                "type": "final",
+                "content": "비교 분석에는 run_id 두 개가 필요합니다. 비교할 run을 알려주세요.",
+            }
+        )
+        return
+
     yield _event({"type": "status", "message": "도구 실행 중..."})
     try:
-        result = await asyncio.wait_for(_call_mcp_tool(tool_name, tool_args), timeout=12)
+        enhanced_tool_args = dict(tool_args)
+        if run_id:
+            enhanced_tool_args["run_id"] = run_id
+        if category:
+            enhanced_tool_args["category"] = category
+        result = await asyncio.wait_for(_call_mcp_tool(tool_name, enhanced_tool_args), timeout=12)
     except TimeoutError:
         yield _event(
             {"type": "error", "message": "응답 지연(12s 초과). MCP 서버 상태를 확인해주세요."}
@@ -615,3 +867,32 @@ async def chat_stream(request: ChatRequest):
             yield item
 
     return StreamingResponse(event_generator(), media_type="application/x-ndjson")
+
+
+@router.post("/ai-stream")
+async def ai_chat_stream(request: AiChatRequest):
+    user_text = _extract_last_user_message(request.messages)
+    run_id = request.run_id
+    category = request.category
+    if not user_text:
+
+        async def error_generator():
+            yield _ai_sse_event({"type": "error", "errorText": "질문을 입력해주세요."})
+            yield _ai_sse_event({"type": "finish"})
+            yield _ai_sse_done()
+
+        return StreamingResponse(
+            error_generator(),
+            media_type="text/event-stream",
+            headers=_ai_sse_headers(),
+        )
+
+    async def event_generator():
+        async for item in _ai_chat_stream(user_text, run_id=run_id, category=category):
+            yield item
+
+    return StreamingResponse(
+        event_generator(),
+        media_type="text/event-stream",
+        headers=_ai_sse_headers(),
+    )

evalvault/adapters/outbound/ops/__init__.py

@@ -0,0 +1,5 @@
+"""Ops report renderers."""
+
+from evalvault.adapters.outbound.ops.report_renderer import render_json, render_markdown
+
+__all__ = ["render_json", "render_markdown"]

evalvault/adapters/outbound/ops/report_renderer.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+import json
+
+from evalvault.domain.entities.ops_report import OpsReport
+from evalvault.domain.entities.stage import StageMetric, StageSummary
+
+
+def render_markdown(report: OpsReport) -> str:
+    lines: list[str] = []
+    lines.append("# Ops Report")
+    lines.append("")
+    lines.extend(_render_run_summary(report.run_summary, report.metadata))
+    lines.append("")
+    lines.extend(_render_ops_kpis(report.ops_kpis))
+    lines.append("")
+    lines.extend(_render_stage_summary(report.stage_summary))
+    lines.append("")
+    lines.extend(_render_bottlenecks(report.bottlenecks))
+    lines.append("")
+    lines.extend(_render_recommendations(report.recommendations))
+    lines.append("")
+    lines.extend(_render_failing_metrics(report.stage_metrics))
+    return "\n".join(lines).strip()
+
+
+def render_json(report: OpsReport) -> str:
+    payload = report.to_dict()
+    return json.dumps(payload, ensure_ascii=True, indent=2)
+
+
+def _render_run_summary(summary: dict[str, object], metadata: dict[str, object]) -> list[str]:
+    run_id = summary.get("run_id", "-")
+    dataset = summary.get("dataset_name", "-")
+    version = summary.get("dataset_version", "-")
+    model = summary.get("model_name", "-")
+    started = summary.get("started_at", "-")
+    finished = summary.get("finished_at", "-")
+    duration = summary.get("duration_seconds", "-")
+    total_cases = summary.get("total_test_cases", "-")
+    pass_rate = summary.get("pass_rate", "-")
+    total_tokens = summary.get("total_tokens", "-")
+    total_cost = summary.get("total_cost_usd", "-")
+
+    lines = [
+        "## Run Summary",
+        f"- run_id: {run_id}",
+        f"- dataset: {dataset} ({version})",
+        f"- model: {model}",
+        f"- started_at: {started}",
+        f"- finished_at: {finished}",
+        f"- duration_seconds: {duration}",
+        f"- total_test_cases: {total_cases}",
+        f"- pass_rate: {pass_rate}",
+        f"- total_tokens: {total_tokens}",
+        f"- total_cost_usd: {total_cost}",
+    ]
+    trace_links: list[str] = []
+    if metadata.get("langfuse_trace_url"):
+        trace_links.append(f"langfuse_trace_url={metadata['langfuse_trace_url']}")
+    if metadata.get("phoenix_trace_url"):
+        trace_links.append(f"phoenix_trace_url={metadata['phoenix_trace_url']}")
+    if trace_links:
+        lines.append(f"- trace_links: {', '.join(trace_links)}")
+    return lines
+
+
+def _render_stage_summary(summary: StageSummary | None) -> list[str]:
+    lines = ["## Stage Summary"]
+    if summary is None:
+        lines.append("- no stage events found")
+        return lines
+    lines.append(f"- total_events: {summary.total_events}")
+    if summary.missing_required_stage_types:
+        missing = ", ".join(summary.missing_required_stage_types)
+        lines.append(f"- missing_required_stage_types: {missing}")
+    if summary.stage_type_counts:
+        lines.append("- stage_type_counts:")
+        for stage_type, count in summary.stage_type_counts.items():
+            lines.append(f"  - {stage_type}: {count}")
+    if summary.stage_type_avg_durations:
+        lines.append("- stage_type_avg_durations_ms:")
+        for stage_type, duration in summary.stage_type_avg_durations.items():
+            lines.append(f"  - {stage_type}: {duration:.3f}")
+    return lines
+
+
+def _render_ops_kpis(kpis: dict[str, object]) -> list[str]:
+    lines = ["## Ops KPIs"]
+    lines.append(f"- total_test_cases: {kpis.get('total_test_cases', '-')}")
+    lines.append(f"- pass_rate: {kpis.get('pass_rate', '-')}")
+    lines.append(f"- failure_rate: {kpis.get('failure_rate', '-')}")
+    lines.append(f"- stage_error_rate: {kpis.get('stage_error_rate', '-')}")
+    lines.append(f"- stage_error_severity: {kpis.get('stage_error_severity', '-')}")
+    lines.append(f"- duration_seconds: {kpis.get('duration_seconds', '-')}")
+    lines.append(f"- total_tokens: {kpis.get('total_tokens', '-')}")
+    lines.append(f"- total_cost_usd: {kpis.get('total_cost_usd', '-')}")
+    lines.append(f"- avg_latency_ms: {kpis.get('avg_latency_ms', '-')}")
+    lines.append(f"- p95_latency_ms: {kpis.get('p95_latency_ms', '-')}")
+    lines.append(f"- avg_tokens_per_case: {kpis.get('avg_tokens_per_case', '-')}")
+    lines.append(f"- avg_cost_per_case_usd: {kpis.get('avg_cost_per_case_usd', '-')}")
+    return lines
+
+
+def _render_bottlenecks(bottlenecks: list[dict[str, object]]) -> list[str]:
+    lines = ["## Ops Signals"]
+    if not bottlenecks:
+        lines.append("- none")
+        return lines
+    for entry in bottlenecks:
+        entry_type = entry.get("type", "unknown")
+        if entry_type == "latency":
+            stage_type = entry.get("stage_type", "-")
+            duration = entry.get("avg_duration_ms", "-")
+            lines.append(f"- latency: {stage_type} avg_duration_ms={duration}")
+        elif entry_type == "missing_stage":
+            stage_type = entry.get("stage_type", "-")
+            lines.append(f"- missing_stage: {stage_type}")
+        else:
+            lines.append(f"- {entry_type}: {entry}")
+    return lines
+
+
+def _render_recommendations(recommendations: list[str]) -> list[str]:
+    lines = ["## Recommendations"]
+    if not recommendations:
+        lines.append("- none")
+        return lines
+    for item in recommendations:
+        lines.append(f"- {item}")
+    return lines
+
+
+def _render_failing_metrics(metrics: list[StageMetric]) -> list[str]:
+    lines = ["## Failing Stage Metrics"]
+    failing = [metric for metric in metrics if metric.passed is False]
+    if not failing:
+        lines.append("- none")
+        return lines
+
+    failing_sorted = sorted(failing, key=_metric_severity, reverse=True)[:20]
+    for metric in failing_sorted:
+        threshold = metric.threshold if metric.threshold is not None else "-"
+        lines.append(
+            f"- {metric.metric_name}: score={metric.score} threshold={threshold} "
+            f"stage_id={metric.stage_id}"
+        )
+    return lines
+
+
+def _metric_severity(metric: StageMetric) -> float:
+    if metric.threshold is None:
+        return 0.0
+    comparison = None
+    if isinstance(metric.evidence, dict):
+        comparison = metric.evidence.get("comparison")
+    if isinstance(comparison, str) and comparison.lower() in {"max", "<=", "le"}:
+        return max(metric.score - metric.threshold, 0.0)
+    return max(metric.threshold - metric.score, 0.0)

evalvault/adapters/outbound/storage/postgres_adapter.py

@@ -1128,6 +1128,96 @@ class PostgreSQLStorageAdapter(BaseSQLStorageAdapter):
             )
         return reports
 
+    def save_ops_report(
+        self,
+        *,
+        report_id: str | None,
+        run_id: str | None,
+        report_type: str,
+        format: str,
+        content: str | None,
+        metadata: dict[str, Any] | None = None,
+        created_at: str | None = None,
+    ) -> str:
+        report_id = report_id or str(uuid.uuid4())
+        if created_at is None:
+            created_at_value = datetime.now(UTC)
+        else:
+            created_at_value = (
+                datetime.fromisoformat(created_at) if isinstance(created_at, str) else created_at
+            )
+
+        with self._get_connection() as conn:
+            conn.execute(
+                """
+                INSERT INTO ops_reports (
+                    report_id, run_id, report_type, format, content, metadata, created_at
+                ) VALUES (%s, %s, %s, %s, %s, %s, %s)
+                ON CONFLICT (report_id) DO UPDATE SET
+                    run_id = EXCLUDED.run_id,
+                    report_type = EXCLUDED.report_type,
+                    format = EXCLUDED.format,
+                    content = EXCLUDED.content,
+                    metadata = EXCLUDED.metadata,
+                    created_at = EXCLUDED.created_at
+                """,
+                (
+                    report_id,
+                    run_id,
+                    report_type,
+                    format,
+                    content,
+                    self._serialize_pipeline_json(metadata),
+                    created_at_value,
+                ),
+            )
+            conn.commit()
+
+        return report_id
+
+    def list_ops_reports(
+        self,
+        *,
+        run_id: str,
+        report_type: str | None = None,
+        format: str | None = None,
+        limit: int = 20,
+    ) -> list[dict[str, Any]]:
+        clauses = ["run_id = %s"]
+        params: list[Any] = [run_id]
+        if report_type:
+            clauses.append("report_type = %s")
+            params.append(report_type)
+        if format:
+            clauses.append("format = %s")
+            params.append(format)
+        params.append(limit)
+
+        query = (
+            "SELECT report_id, run_id, report_type, format, content, metadata, created_at "
+            "FROM ops_reports WHERE " + " AND ".join(clauses) + " ORDER BY created_at DESC LIMIT %s"
+        )
+
+        with self._get_connection() as conn:
+            rows = conn.execute(query, tuple(params)).fetchall()
+
+        reports: list[dict[str, Any]] = []
+        for row in rows:
+            reports.append(
+                {
+                    "report_id": row["report_id"],
+                    "run_id": row["run_id"],
+                    "report_type": row["report_type"],
+                    "format": row["format"],
+                    "content": row["content"],
+                    "metadata": self._deserialize_json(row["metadata"]),
+                    "created_at": row["created_at"].isoformat()
+                    if isinstance(row["created_at"], datetime)
+                    else row["created_at"],
+                }
+            )
+        return reports
+
     def list_pipeline_results(self, limit: int = 50) -> list[dict[str, Any]]:
         """파이프라인 분석 결과 목록을 조회합니다."""
         query = """

evalvault/adapters/outbound/storage/postgres_schema.sql

@@ -241,6 +241,19 @@ CREATE TABLE IF NOT EXISTS analysis_reports (
 CREATE INDEX IF NOT EXISTS idx_reports_run_id ON analysis_reports(run_id);
 CREATE INDEX IF NOT EXISTS idx_reports_experiment_id ON analysis_reports(experiment_id);
 
+-- Ops reports table
+CREATE TABLE IF NOT EXISTS ops_reports (
+    report_id UUID PRIMARY KEY,
+    run_id UUID REFERENCES evaluation_runs(run_id) ON DELETE SET NULL,
+    report_type VARCHAR(50) NOT NULL,
+    format VARCHAR(20) NOT NULL,
+    content TEXT,
+    metadata JSONB,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX IF NOT EXISTS idx_ops_reports_run_id ON ops_reports(run_id);
+
 -- Analysis pipeline results table
 CREATE TABLE IF NOT EXISTS pipeline_results (
     result_id UUID PRIMARY KEY,

evalvault/adapters/outbound/storage/schema.sql

@@ -271,6 +271,20 @@ CREATE TABLE IF NOT EXISTS analysis_reports (
 CREATE INDEX IF NOT EXISTS idx_reports_run_id ON analysis_reports(run_id);
 CREATE INDEX IF NOT EXISTS idx_reports_experiment_id ON analysis_reports(experiment_id);
 
+-- Ops reports table
+CREATE TABLE IF NOT EXISTS ops_reports (
+    report_id TEXT PRIMARY KEY,
+    run_id TEXT,
+    report_type TEXT NOT NULL,  -- 'ops_report', 'ops_snapshot'
+    format TEXT NOT NULL,  -- 'markdown', 'json'
+    content TEXT,  -- Report content (markdown/json) or file path
+    metadata TEXT,  -- JSON metadata
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (run_id) REFERENCES evaluation_runs(run_id) ON DELETE SET NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_ops_reports_run_id ON ops_reports(run_id);
+
 -- Analysis pipeline results table
 CREATE TABLE IF NOT EXISTS pipeline_results (
     result_id TEXT PRIMARY KEY,

evalvault/adapters/outbound/storage/sqlite_adapter.py

@@ -1211,6 +1211,83 @@ class SQLiteStorageAdapter(BaseSQLStorageAdapter):
             )
         return reports
 
+    def save_ops_report(
+        self,
+        *,
+        report_id: str | None,
+        run_id: str | None,
+        report_type: str,
+        format: str,
+        content: str | None,
+        metadata: dict[str, Any] | None = None,
+        created_at: str | None = None,
+    ) -> str:
+        report_id = report_id or str(uuid.uuid4())
+        created_at = created_at or datetime.now().isoformat()
+
+        with self._get_connection() as conn:
+            conn = cast(Any, conn)
+            conn.execute(
+                """
+                INSERT OR REPLACE INTO ops_reports (
+                    report_id, run_id, report_type, format, content, metadata, created_at
+                ) VALUES (?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    report_id,
+                    run_id,
+                    report_type,
+                    format,
+                    content,
+                    self._serialize_json(metadata),
+                    created_at,
+                ),
+            )
+            conn.commit()
+
+        return report_id
+
+    def list_ops_reports(
+        self,
+        *,
+        run_id: str,
+        report_type: str | None = None,
+        format: str | None = None,
+        limit: int = 20,
+    ) -> list[dict[str, Any]]:
+        query = (
+            "SELECT report_id, run_id, report_type, format, content, metadata, created_at "
+            "FROM ops_reports WHERE run_id = ?"
+        )
+        params: list[Any] = [run_id]
+        if report_type:
+            query += " AND report_type = ?"
+            params.append(report_type)
+        if format:
+            query += " AND format = ?"
+            params.append(format)
+        query += " ORDER BY created_at DESC LIMIT ?"
+        params.append(limit)
+
+        with self._get_connection() as conn:
+            conn = cast(Any, conn)
+            rows = conn.execute(query, tuple(params)).fetchall()
+
+        reports: list[dict[str, Any]] = []
+        for row in rows:
+            reports.append(
+                {
+                    "report_id": row["report_id"],
+                    "run_id": row["run_id"],
+                    "report_type": row["report_type"],
+                    "format": row["format"],
+                    "content": row["content"],
+                    "metadata": self._deserialize_json(row["metadata"]),
+                    "created_at": row["created_at"],
+                }
+            )
+        return reports
+
     def list_pipeline_results(self, limit: int = 50) -> list[dict[str, Any]]:
         """파이프라인 분석 결과 목록을 조회합니다."""
         query = """

evalvault/domain/entities/ops_report.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from evalvault.domain.entities.stage import StageMetric, StageSummary
+
+
+@dataclass
+class OpsReport:
+    run_summary: dict[str, Any]
+    ops_kpis: dict[str, Any]
+    stage_summary: StageSummary | None
+    stage_metrics: list[StageMetric]
+    bottlenecks: list[dict[str, Any]]
+    recommendations: list[str]
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "run_summary": self.run_summary,
+            "ops_kpis": self.ops_kpis,
+            "stage_summary": _stage_summary_to_dict(self.stage_summary),
+            "stage_metrics": [metric.to_dict() for metric in self.stage_metrics],
+            "bottlenecks": self.bottlenecks,
+            "recommendations": self.recommendations,
+            "metadata": self.metadata,
+        }
+
+
+def _stage_summary_to_dict(summary: StageSummary | None) -> dict[str, Any] | None:
+    if summary is None:
+        return None
+    return {
+        "run_id": summary.run_id,
+        "total_events": summary.total_events,
+        "stage_type_counts": summary.stage_type_counts,
+        "stage_type_avg_durations": summary.stage_type_avg_durations,
+        "missing_required_stage_types": summary.missing_required_stage_types,
+    }

evalvault/domain/services/ops_report_service.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+from typing import Any
+
+from evalvault.config.langfuse_support import get_langfuse_trace_url
+from evalvault.config.phoenix_support import get_phoenix_trace_url
+from evalvault.domain.entities.ops_report import OpsReport
+from evalvault.domain.entities.stage import StageEvent, StageMetric, StageSummary
+from evalvault.domain.services.stage_metric_guide_service import StageMetricGuideService
+from evalvault.domain.services.stage_metric_service import StageMetricService
+from evalvault.domain.services.stage_summary_service import StageSummaryService
+from evalvault.ports.outbound.stage_storage_port import StageStoragePort
+from evalvault.ports.outbound.storage_port import StoragePort
+
+
+class OpsReportService:
+    """Build an operational report for an evaluation run."""
+
+    def __init__(
+        self,
+        *,
+        metric_service: StageMetricService | None = None,
+        summary_service: StageSummaryService | None = None,
+        guide_service: StageMetricGuideService | None = None,
+    ) -> None:
+        self._metric_service = metric_service or StageMetricService()
+        self._summary_service = summary_service or StageSummaryService()
+        self._guide_service = guide_service or StageMetricGuideService()
+
+    def build_report(
+        self,
+        run_id: str,
+        *,
+        storage: StoragePort,
+        stage_storage: StageStoragePort,
+    ) -> OpsReport:
+        run = storage.get_run(run_id)
+        run_summary = run.to_summary_dict()
+        phoenix_trace_url = get_phoenix_trace_url(run.tracker_metadata)
+        langfuse_trace_url = get_langfuse_trace_url(run.tracker_metadata)
+
+        events = stage_storage.list_stage_events(run_id)
+        stage_summary = self._summarize_events(events)
+
+        stage_metrics = stage_storage.list_stage_metrics(run_id)
+        if not stage_metrics and events:
+            stage_metrics = self._metric_service.build_metrics(events)
+
+        bottlenecks = self._build_bottlenecks(stage_summary)
+        recommendations = self._build_recommendations(stage_metrics)
+
+        ops_kpis = self._build_ops_kpis(run, events)
+
+        metadata = {
+            "phoenix_trace_url": phoenix_trace_url,
+            "langfuse_trace_url": langfuse_trace_url,
+        }
+
+        return OpsReport(
+            run_summary=run_summary,
+            ops_kpis=ops_kpis,
+            stage_summary=stage_summary,
+            stage_metrics=stage_metrics,
+            bottlenecks=bottlenecks,
+            recommendations=recommendations,
+            metadata=metadata,
+        )
+
+    def _summarize_events(self, events: list[StageEvent]) -> StageSummary | None:
+        if not events:
+            return None
+        return self._summary_service.summarize(events)
+
+    def _build_bottlenecks(self, summary: StageSummary | None) -> list[dict[str, Any]]:
+        if summary is None:
+            return []
+        bottlenecks: list[dict[str, Any]] = []
+
+        for stage_type in summary.missing_required_stage_types:
+            bottlenecks.append(
+                {
+                    "type": "missing_stage",
+                    "stage_type": stage_type,
+                    "detail": "required stage missing",
+                }
+            )
+
+        durations = summary.stage_type_avg_durations
+        if durations:
+            top = sorted(durations.items(), key=lambda item: item[1], reverse=True)[:3]
+            for stage_type, duration in top:
+                bottlenecks.append(
+                    {
+                        "type": "latency",
+                        "stage_type": stage_type,
+                        "avg_duration_ms": round(duration, 3),
+                    }
+                )
+        return bottlenecks
+
+    def _build_recommendations(self, metrics: list[StageMetric]) -> list[str]:
+        if not metrics:
+            return []
+        guides = self._guide_service.build_guides(metrics)
+        recommendations: list[str] = []
+        for guide in guides:
+            top_action = guide.top_action
+            if top_action is None:
+                continue
+            hint = top_action.implementation_hint or top_action.description
+            label = f"[{guide.priority.value}] {guide.component.value}"
+            if hint:
+                recommendations.append(f"{label}: {top_action.title} - {hint}")
+            else:
+                recommendations.append(f"{label}: {top_action.title}")
+        return recommendations
+
+    def _build_ops_kpis(self, run, events: list[StageEvent]) -> dict[str, Any]:
+        total_cases = run.total_test_cases
+        latencies = [r.latency_ms for r in run.results if r.latency_ms]
+        tokens_used = [r.tokens_used for r in run.results if r.tokens_used]
+        costs = [r.cost_usd for r in run.results if r.cost_usd is not None]
+
+        avg_latency = _average(latencies)
+        p95_latency = _percentile(latencies, 0.95)
+        avg_tokens = _average(tokens_used)
+        avg_cost = _average(costs)
+        pass_rate = run.pass_rate
+        failure_rate = None if pass_rate is None else max(0.0, 1.0 - pass_rate)
+
+        error_rate = _stage_error_rate(events)
+        error_severity = _stage_error_severity(error_rate)
+
+        return {
+            "total_test_cases": total_cases,
+            "pass_rate": pass_rate,
+            "failure_rate": failure_rate,
+            "stage_error_rate": error_rate,
+            "stage_error_severity": error_severity,
+            "duration_seconds": run.duration_seconds,
+            "total_tokens": run.total_tokens,
+            "total_cost_usd": run.total_cost_usd,
+            "avg_latency_ms": avg_latency,
+            "p95_latency_ms": p95_latency,
+            "avg_tokens_per_case": avg_tokens,
+            "avg_cost_per_case_usd": avg_cost,
+        }
+
+
+def _average(values: list[float | int]) -> float | None:
+    if not values:
+        return None
+    return float(sum(values)) / len(values)
+
+
+def _percentile(values: list[float | int], ratio: float) -> float | None:
+    if not values:
+        return None
+    if ratio <= 0:
+        return float(min(values))
+    if ratio >= 1:
+        return float(max(values))
+    sorted_values = sorted(values)
+    index = int(round((len(sorted_values) - 1) * ratio))
+    return float(sorted_values[index])
+
+
+def _stage_error_rate(events: list[StageEvent]) -> float | None:
+    if not events:
+        return None
+    total = len(events)
+    failure_statuses = {"failed", "error", "timeout", "aborted"}
+    success_statuses = {"success", "ok", "completed", "pass"}
+    failures = 0
+    for event in events:
+        status = str(event.status or "").strip().lower()
+        if status in failure_statuses:
+            failures += 1
+            continue
+        if status and status not in success_statuses:
+            failures += 1
+    return failures / total
+
+
+def _stage_error_severity(rate: float | None) -> str | None:
+    if rate is None:
+        return None
+    if rate >= 0.05:
+        return "critical"
+    if rate >= 0.02:
+        return "warning"
+    return "ok"

{evalvault-1.73.2.dist-info → evalvault-1.74.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evalvault
-Version: 1.73.2
+Version: 1.74.0
 Summary: RAG evaluation system using Ragas with Phoenix/Langfuse tracing
 Project-URL: Homepage, https://github.com/ntts9990/EvalVault
 Project-URL: Documentation, https://github.com/ntts9990/EvalVault#readme

{evalvault-1.73.2.dist-info → evalvault-1.74.0.dist-info}/RECORD

@@ -5,12 +5,12 @@ evalvault/mkdocs_helpers.py,sha256=1AKVQ1W2_VO4qclhfyefyU9Dz1Hzkh1DWDwsFMe24jc,3
 evalvault/adapters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 evalvault/adapters/inbound/__init__.py,sha256=SG1svel1PwqetnqVpKFLSv612_WwGwLTbFpYgwk6FMw,166
 evalvault/adapters/inbound/api/__init__.py,sha256=LeVVttCA3tLKoHA2PO4z3y8VkfVcf3Bq8CZSzo91lf4,34
-evalvault/adapters/inbound/api/adapter.py,sha256=Igg2grCUxQzMuvDOAhBK08wY0nxjmnvnaGS5rLVF3i4,83388
+evalvault/adapters/inbound/api/adapter.py,sha256=B0kLz4aEsZ-WeCQlUKSanSz6r9gedr_JmiRIaso3HoA,84659
 evalvault/adapters/inbound/api/main.py,sha256=QgLxzHEy7aycGKIFLtN12tWTjnpWLtQ2XDXKV_2FDvg,7531
 evalvault/adapters/inbound/api/routers/__init__.py,sha256=q07_YF9TnBl68bqcRCvhPU4-zRTyvmPoHVehwO6W7QM,19
 evalvault/adapters/inbound/api/routers/benchmark.py,sha256=yevntbZcNtMvbVODsITUBgR1Ka4pdFQrXBJJ4K4Jyr4,4477
 evalvault/adapters/inbound/api/routers/calibration.py,sha256=ZnJSEW8hV-94S95lU_nDmzcLyaUoH1suM3sFUpJ3w5k,4130
-evalvault/adapters/inbound/api/routers/chat.py,sha256=hCA6rWr5GT_gCqu75uCqYwy2gOEUd85mlcc5y-ruFTY,20661
+evalvault/adapters/inbound/api/routers/chat.py,sha256=lQfJTPPcA3H3IzxNpYSTBBAy-gSxCYYmDfEr4hCFp9M,30676
 evalvault/adapters/inbound/api/routers/config.py,sha256=LygN0fVMr8NFtj5zuQXnVFhoafx56Txa98vpwtPa4Jc,4104
 evalvault/adapters/inbound/api/routers/domain.py,sha256=RsR7GIFMjccDN7vpG1uDyk9n1DnCTH18JDGAX7o4Qqc,3648
 evalvault/adapters/inbound/api/routers/knowledge.py,sha256=yb_e7OEPtwldOAzHTGiWe7jShHw2JdpOFnzGPMceRsg,7109
@@ -181,6 +181,8 @@ evalvault/adapters/outbound/nlp/korean/korean_evaluation.py,sha256=Mxwu3zhtdm8Te
 evalvault/adapters/outbound/nlp/korean/korean_stopwords.py,sha256=UemEFCJudg2EpsHg8uU2eR-iCh34kw4ZSVCRvnEC6a4,4293
 evalvault/adapters/outbound/nlp/korean/toolkit.py,sha256=EYGpd89ilpn4Wg5t8pALYt4Qi0aDHYOfXGuYbQx7do0,4246
 evalvault/adapters/outbound/nlp/korean/toolkit_factory.py,sha256=x3v-AAkVInOabC4PtOtStsZrFnHun0IOqZDyQGaQVm8,586
+evalvault/adapters/outbound/ops/__init__.py,sha256=_QiDVPuiYWkIwW_ELEVKD_v6dLojjyvIJWs4qVNxehw,164
+evalvault/adapters/outbound/ops/report_renderer.py,sha256=mezVKdIsnJSNvBW6xkhpNG3MOFXHZLZspmHk5o-e8Cg,6354
 evalvault/adapters/outbound/phoenix/sync_service.py,sha256=i6gHpNiZXKQ5yzV9B2TPb-P1N45k_Ck5ruzh3oqp4d8,9122
 evalvault/adapters/outbound/report/__init__.py,sha256=8VeMrfj63mDR-xUHct-drNNBA5M-m-B7sgC1qUJF7g4,660
 evalvault/adapters/outbound/report/ci_report_formatter.py,sha256=5YD8BwtOjLnHcNbbG0HJziOifD9BDhBtZT1oItd6zJE,1233
@@ -193,10 +195,10 @@ evalvault/adapters/outbound/retriever/graph_rag_adapter.py,sha256=xTI7uMFp4WKstg
 evalvault/adapters/outbound/storage/__init__.py,sha256=n5R6thAPTx1leSwv6od6nBWcLWFa-UYD6cOLzN89T8I,614
 evalvault/adapters/outbound/storage/base_sql.py,sha256=NHHlBBhoDq8zbLfIvRoOswjB97Tro1sRE3cmbR8QnV8,58380
 evalvault/adapters/outbound/storage/benchmark_storage_adapter.py,sha256=Qgf9xSSIkYQRpG4uLzcUdoYO9LTQDQ4tFRkkMYer-WA,9803
-evalvault/adapters/outbound/storage/postgres_adapter.py,sha256=pyIPMliC7bsIXoRQ8l6a_svQUJ7or1XFxERT-DFJuf4,56854
-evalvault/adapters/outbound/storage/postgres_schema.sql,sha256=WQNeOUm9UYWteh6k-PLrIhgkagIv_UPGCIi4nGzP9VA,11916
-evalvault/adapters/outbound/storage/schema.sql,sha256=ERgupB1bH1xqQr56GZHyVGfcUqr_7NGfg7U6f69W29Y,13746
-evalvault/adapters/outbound/storage/sqlite_adapter.py,sha256=9Z2A4C5QZEcQiQVt4h7fJMIbPSA3lV_u2bd2nXqDdOA,60491
+evalvault/adapters/outbound/storage/postgres_adapter.py,sha256=SZb4Dx2ZYNu-pDqX4rVgT4bRfJ2cQ75DXcWIIEu8aws,59933
+evalvault/adapters/outbound/storage/postgres_schema.sql,sha256=E-HMnGjrB05n2Z-zDq1kQzYSpYlPwg2Egm1e3MSRB7E,12335
+evalvault/adapters/outbound/storage/schema.sql,sha256=LEtrKFpa1SbIBN-igkNiQXJqVLGp-liX-6KR_sZAMwM,14283
+evalvault/adapters/outbound/storage/sqlite_adapter.py,sha256=Gsco1g_N1RHyRzIQIwelR-PU7kx57djHOc_hKEu_gCc,62940
 evalvault/adapters/outbound/tracer/__init__.py,sha256=xrvQQuAvF_UI02mKLMV7GTrG3zn836n5zwCRrrmhq_U,1054
 evalvault/adapters/outbound/tracer/open_rag_log_handler.py,sha256=aq96FIWD-bBaSkq-bygWhQArC9LWghSwi-S03Mga0mI,2827
 evalvault/adapters/outbound/tracer/open_rag_trace_adapter.py,sha256=X_dlqUEwck_MQRGC-euEN73L2Ikcd5eZlgtKK24nDFY,5453
@@ -235,6 +237,7 @@ evalvault/domain/entities/kg.py,sha256=8awN1M4vxAGQZk_ZG8i2CXKTizQ8FA1VCLhUWHZq0
 evalvault/domain/entities/memory.py,sha256=bfS75q8K8_jNrB7IYh4mjP8Lkyj-I0TVsmHCP0FuICw,8423
 evalvault/domain/entities/method.py,sha256=a3jZi7SjcpK3HeVyVwQkUMwpnmg2RbxCnH4NqYPLCOI,1157
 evalvault/domain/entities/multiturn.py,sha256=V9ay30rix6zxNcDRXeLudMgikC1b4f3kt01Hj2ZH7wE,2012
+evalvault/domain/entities/ops_report.py,sha256=89x1UKdFb6oxpsQb4pfED69dcCqnyrzBuOo42dcbgho,1343
 evalvault/domain/entities/prompt.py,sha256=lQlRnHEKY69GWTC-cUIu0DMuPfJ9UWm6Sm4KTNjVwfY,2920
 evalvault/domain/entities/prompt_suggestion.py,sha256=Ep_XSjdYUj7pFSCMyeeZKs8yTnp74AVx05Zqr7829PE,1243
 evalvault/domain/entities/rag_trace.py,sha256=sZgnkG4fK6KOe3Np6TYAZ_tPnsRbOmucDSQns35U1n4,11868
@@ -292,6 +295,7 @@ evalvault/domain/services/memory_aware_evaluator.py,sha256=vTiYoxiMfZ_CMjSBjqwkB
 evalvault/domain/services/memory_based_analysis.py,sha256=oh2irCy3le7fWiTtL31SMEhPyu7fyBVz-giO2hlNifE,4499
 evalvault/domain/services/method_runner.py,sha256=pABqKZeaALpWZYDfzAbd-VOZt2djQggRNIPuuPQeUSw,3571
 evalvault/domain/services/multiturn_evaluator.py,sha256=fipi5hEyidq_cnGGr0GpvoprLtjm6dHLuAkSotbT3YA,7202
+evalvault/domain/services/ops_report_service.py,sha256=FNMN0GpaA3rnnbYs-5xY1ReXzGWaQG0t3ZN0eoeQiRc,7018
 evalvault/domain/services/ops_snapshot_service.py,sha256=1CqJN2p3tM6SgzLCZKcVEM213fd1cDGexTRPG_3e59w,5138
 evalvault/domain/services/pipeline_orchestrator.py,sha256=yriVlEVZYDtt0Vwt4Ae6xyW1H6Dj4Hxdn8XQSvQNSoQ,19436
 evalvault/domain/services/pipeline_template_registry.py,sha256=Gg0k-Sj3MNDBjckhI2gQRCNUsqIz0nnjPuswW5kUdBw,29567
@@ -355,8 +359,8 @@ evalvault/reports/__init__.py,sha256=Bb1X4871msAN8I6PM6nKGED3psPwZt88hXZBAOdH06Y
 evalvault/reports/release_notes.py,sha256=pZj0PBFT-4F_Ty-Kv5P69BuoOnmTCn4kznDcORFJd0w,4011
 evalvault/scripts/__init__.py,sha256=NwEeIFQbkX4ml2R_PhtIoNtArDSX_suuoymgG_7Kwso,89
 evalvault/scripts/regression_runner.py,sha256=SxZori5BZ8jVQ057Mf5V5FPgIVDccrV5oRONmnhuk8w,8438
-evalvault-1.73.2.dist-info/METADATA,sha256=j2HvoSJag0ISVSoJnZN5iQ_QPQy_djdEDgfy27kHepI,26218
-evalvault-1.73.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-evalvault-1.73.2.dist-info/entry_points.txt,sha256=Oj9Xc5gYcyUYYNmQfWI8NYGw7nN-3M-h2ipHIMlVn6o,65
-evalvault-1.73.2.dist-info/licenses/LICENSE.md,sha256=3RNWY4jjtrQ_yYa-D-7I3XO12Ti7YzxsLV_dpykujvo,11358
-evalvault-1.73.2.dist-info/RECORD,,
+evalvault-1.74.0.dist-info/METADATA,sha256=ASZ_QeCeh2pyJ3eBUfu0zDKbIwDbvMp9JzL6KVvjWqk,26218
+evalvault-1.74.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+evalvault-1.74.0.dist-info/entry_points.txt,sha256=Oj9Xc5gYcyUYYNmQfWI8NYGw7nN-3M-h2ipHIMlVn6o,65
+evalvault-1.74.0.dist-info/licenses/LICENSE.md,sha256=3RNWY4jjtrQ_yYa-D-7I3XO12Ti7YzxsLV_dpykujvo,11358
+evalvault-1.74.0.dist-info/RECORD,,