evalvault 1.66.0__py3-none-any.whl → 1.67.0__py3-none-any.whl

evalvault/adapters/inbound/api/main.py

@@ -63,6 +63,12 @@ async def lifespan(app: FastAPI):
     # Startup: Initialize adapter
     adapter = create_adapter()
     app.state.adapter = adapter
+    try:
+        from evalvault.adapters.inbound.api.routers.chat import warm_rag_index
+
+        await warm_rag_index()
+    except Exception as exc:
+        logger.warning("RAG preload failed: %s", exc)
     yield
     # Shutdown: Cleanup if necessary
     pass

evalvault/adapters/inbound/api/routers/chat.py

@@ -1,11 +1,14 @@
 from __future__ import annotations
 
 import asyncio
+import hashlib
 import json
+import logging
 import os
 import re
 import time
 from collections.abc import AsyncGenerator
+from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
 
@@ -16,9 +19,15 @@ from pydantic import BaseModel, Field
 
 router = APIRouter(tags=["chat"])
 
+logger = logging.getLogger(__name__)
+
 MCP_URL = os.getenv("EVALVAULT_MCP_URL", "http://localhost:8000/api/v1/mcp")
 MCP_TOKEN = os.getenv("EVALVAULT_MCP_TOKEN", "mcp-local-dev-token")
 
+USER_GUIDE_PATH = Path(os.getenv("EVALVAULT_RAG_USER_GUIDE", "docs/guides/USER_GUIDE.md"))
+RAG_INDEX_DIR = Path(os.getenv("EVALVAULT_RAG_INDEX_DIR", "data/rag"))
+RAG_INDEX_PATH = RAG_INDEX_DIR / "user_guide_bm25.json"
+
 _RAG_RETRIEVER = None
 _RAG_DOCS_COUNT = 0
 _RAG_TEXTS: list[str] = []
@@ -129,25 +138,98 @@ def _summarize_result(tool_name: str, payload: dict[str, Any]) -> str:
     return str(payload)
 
 
-def _load_text_files(root: Path, extensions: tuple[str, ...], limit: int) -> list[str]:
-    texts: list[str] = []
-    for path in root.rglob("*"):
-        if not path.is_file():
-            continue
-        if path.suffix.lower() not in extensions:
-            continue
-        if limit and len(texts) >= limit:
-            break
-        try:
-            content = path.read_text(encoding="utf-8")
-        except Exception:
-            continue
-        if content.strip():
-            texts.append(content)
-    return texts
+def _load_user_guide_text() -> str | None:
+    if not USER_GUIDE_PATH.exists():
+        logger.warning("USER_GUIDE.md not found at %s", USER_GUIDE_PATH)
+        return None
+    try:
+        content = USER_GUIDE_PATH.read_text(encoding="utf-8")
+    except Exception as exc:
+        logger.warning("Failed to read USER_GUIDE.md: %s", exc)
+        return None
+    if not content.strip():
+        return None
+    return content
+
+
+def _hash_text(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def _chunk_user_guide(content: str, chunk_limit: int) -> list[str]:
+    try:
+        from evalvault.adapters.outbound.nlp.korean.document_chunker import ParagraphChunker
+        from evalvault.adapters.outbound.nlp.korean.kiwi_tokenizer import KiwiTokenizer
+
+        tokenizer = KiwiTokenizer()
+        chunker = ParagraphChunker(tokenizer=tokenizer, chunk_size=450, overlap_tokens=80)
+        chunks = [
+            chunk.text
+            for chunk in chunker.chunk_with_metadata(content, source=str(USER_GUIDE_PATH))
+        ]
+        if chunk_limit > 0:
+            return chunks[:chunk_limit]
+        return chunks
+    except Exception as exc:
+        logger.warning("Failed to chunk USER_GUIDE.md, using fallback split: %s", exc)
+        paragraphs = [block.strip() for block in content.split("\n\n") if block.strip()]
+        if chunk_limit > 0:
+            return paragraphs[:chunk_limit]
+        return paragraphs
 
 
-async def _get_rag_retriever():
+def _build_bm25_tokens(texts: list[str]) -> list[list[str]]:
+    try:
+        from evalvault.adapters.outbound.nlp.korean.kiwi_tokenizer import KiwiTokenizer
+
+        tokenizer = KiwiTokenizer()
+        tokens = []
+        for text in texts:
+            doc_tokens = tokenizer.tokenize(text)
+            if not doc_tokens:
+                doc_tokens = re.findall(r"[A-Za-z0-9가-힣]+", text)
+            tokens.append(doc_tokens)
+        return tokens
+    except Exception as exc:
+        logger.warning("Failed to tokenize with Kiwi, using regex: %s", exc)
+        return [re.findall(r"[A-Za-z0-9가-힣]+", text) for text in texts]
+
+
+def _load_bm25_index() -> dict[str, Any] | None:
+    if not RAG_INDEX_PATH.exists():
+        return None
+    try:
+        payload = json.loads(RAG_INDEX_PATH.read_text(encoding="utf-8"))
+    except Exception as exc:
+        logger.warning("Failed to read BM25 index: %s", exc)
+        return None
+    if not isinstance(payload, dict):
+        return None
+    return payload
+
+
+def _save_bm25_index(payload: dict[str, Any]) -> None:
+    RAG_INDEX_DIR.mkdir(parents=True, exist_ok=True)
+    RAG_INDEX_PATH.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def _build_bm25_index(content: str, chunk_limit: int) -> dict[str, Any] | None:
+    chunks = _chunk_user_guide(content, chunk_limit)
+    if not chunks:
+        return None
+    tokens = _build_bm25_tokens(chunks)
+    return {
+        "version": 1,
+        "source": str(USER_GUIDE_PATH),
+        "source_hash": _hash_text(content),
+        "chunk_limit": chunk_limit,
+        "created_at": datetime.now(UTC).isoformat(),
+        "documents": chunks,
+        "tokens": tokens,
+    }
+
+
+async def _get_rag_retriever() -> tuple[Any | None, int]:
     global _RAG_RETRIEVER
     global _RAG_DOCS_COUNT
     global _RAG_TEXTS
@@ -156,50 +238,51 @@ async def _get_rag_retriever():
     if _RAG_RETRIEVER is not None:
         return _RAG_RETRIEVER, _RAG_DOCS_COUNT
 
-    if not _RAG_INITIALIZED:
-        docs_root = Path(os.getenv("EVALVAULT_RAG_DOCS", "docs"))
-        src_root = Path(os.getenv("EVALVAULT_RAG_SRC", "src"))
-        docs_limit = int(os.getenv("EVALVAULT_RAG_DOCS_LIMIT", "120"))
-        src_limit = int(os.getenv("EVALVAULT_RAG_SRC_LIMIT", "120"))
-
-        texts: list[str] = []
-        if docs_root.exists():
-            texts.extend(_load_text_files(docs_root, (".md", ".txt"), docs_limit))
-        if src_root.exists():
-            texts.extend(_load_text_files(src_root, (".py",), src_limit))
+    user_guide_limit = int(os.getenv("EVALVAULT_RAG_USER_GUIDE_LIMIT", "80"))
+    content = _load_user_guide_text()
+    if content is None:
+        return None, 0
+    source_hash = _hash_text(content)
+
+    index_payload = _load_bm25_index()
+    if index_payload is None or index_payload.get("source_hash") != source_hash:
+        index_payload = _build_bm25_index(content, user_guide_limit)
+        if index_payload is None:
+            return None, 0
+        _save_bm25_index(index_payload)
+
+    documents = index_payload.get("documents")
+    tokens = index_payload.get("tokens")
+    if not isinstance(documents, list) or not isinstance(tokens, list):
+        return None, 0
 
-        _RAG_TEXTS = texts
-        _RAG_DOCS_COUNT = len(texts)
-        _RAG_INITIALIZED = True
+    _RAG_TEXTS = documents
+    _RAG_DOCS_COUNT = len(documents)
+    _RAG_INITIALIZED = True
 
     if not _RAG_TEXTS:
         return None, 0
 
-    from evalvault.adapters.outbound.llm.ollama_adapter import OllamaAdapter
-    from evalvault.adapters.outbound.nlp.korean.toolkit_factory import try_create_korean_toolkit
-    from evalvault.config.settings import Settings
-
-    settings = Settings()
-    ollama_adapter = OllamaAdapter(settings)
-    toolkit = try_create_korean_toolkit()
-    if toolkit is None:
-        return None, 0
+    from evalvault.adapters.outbound.nlp.korean.bm25_retriever import KoreanBM25Retriever
+    from evalvault.adapters.outbound.nlp.korean.kiwi_tokenizer import KiwiTokenizer
 
-    use_hybrid = os.getenv("EVALVAULT_RAG_USE_HYBRID", "true").lower() == "true"
-    retriever = toolkit.build_retriever(
-        documents=_RAG_TEXTS,
-        use_hybrid=use_hybrid,
-        ollama_adapter=ollama_adapter if use_hybrid else None,
-        embedding_profile=os.getenv("EVALVAULT_RAG_EMBEDDING_PROFILE", "dev"),
-        verbose=False,
-    )
-    if retriever is None:
-        return None, 0
+    tokenizer = KiwiTokenizer()
+    retriever = KoreanBM25Retriever(tokenizer=tokenizer)
+    retriever.index(list(_RAG_TEXTS))
+    if tokens and len(tokens) == len(_RAG_TEXTS):
+        retriever._tokenized_docs = tokens
 
     _RAG_RETRIEVER = retriever
     return retriever, _RAG_DOCS_COUNT
 
 
+async def warm_rag_index() -> None:
+    try:
+        await _get_rag_retriever()
+    except Exception as exc:
+        logger.warning("RAG preload failed: %s", exc)
+
+
 async def _direct_chat_answer(user_text: str) -> str | None:
     payload = {
         "model": os.getenv("OLLAMA_CHAT_MODEL", "gpt-oss-safeguard:20b"),
@@ -351,15 +434,17 @@ async def _resolve_tool_with_llm(user_text: str) -> dict[str, Any] | None:
 
 
 def _extract_json_content(result: Any) -> dict[str, Any] | None:
-    if isinstance(result, dict) and isinstance(result.get("structuredContent"), dict):
-        return result.get("structuredContent")
-
-    if hasattr(result, "structuredContent"):
-        payload = result.structuredContent
-        if isinstance(payload, dict):
-            return payload
+    if isinstance(result, dict):
+        structured = result.get("structuredContent")
+        if isinstance(structured, dict):
+            return structured
+    else:
+        if hasattr(result, "structuredContent"):
+            payload = result.structuredContent
+            if isinstance(payload, dict):
+                return payload
 
-    if hasattr(result, "content"):
+    if not isinstance(result, dict) and hasattr(result, "content"):
         content = result.content
     elif isinstance(result, dict):
         content = result.get("content")
@@ -385,17 +470,6 @@ def _extract_json_content(result: Any) -> dict[str, Any] | None:
                         return None
                     if isinstance(parsed, dict):
                         return parsed
-        else:
-            item_type = getattr(item, "type", None)
-            if item_type == "text":
-                text = getattr(item, "text", None)
-                if isinstance(text, str):
-                    try:
-                        parsed = json.loads(text)
-                    except Exception:
-                        return None
-                    if isinstance(parsed, dict):
-                        return parsed
     return None
 
 

evalvault/adapters/inbound/api/routers/mcp.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+import json
+from dataclasses import asdict
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+from pydantic import BaseModel, Field
+
+from evalvault.adapters.inbound.mcp import tools as mcp_tools
+from evalvault.config.settings import Settings, get_settings
+
+router = APIRouter(tags=["mcp"])
+
+
+class JsonRpcRequest(BaseModel):
+    jsonrpc: str = Field("2.0", pattern=r"^2\.0$")
+    id: int | str | None = None
+    method: str
+    params: dict[str, Any] | None = None
+
+
+def _normalize_tokens(raw_tokens: str | None) -> set[str]:
+    if not raw_tokens:
+        return set()
+    return {token.strip() for token in raw_tokens.split(",") if token.strip()}
+
+
+def _require_mcp_token(
+    request: Request,
+    settings: Settings = Depends(get_settings),
+) -> None:
+    if not settings.mcp_enabled:
+        raise HTTPException(status_code=404, detail="MCP is disabled")
+    tokens = _normalize_tokens(settings.mcp_auth_tokens) or _normalize_tokens(
+        settings.api_auth_tokens
+    )
+    if not tokens:
+        raise HTTPException(status_code=401, detail="MCP auth tokens are required")
+    auth_header = request.headers.get("Authorization", "")
+    if not auth_header.lower().startswith("bearer "):
+        raise HTTPException(status_code=401, detail="Invalid or missing MCP token")
+    token = auth_header[7:].strip()
+    if token not in tokens:
+        raise HTTPException(status_code=401, detail="Invalid or missing MCP token")
+
+
+def _tool_registry() -> dict[str, Any]:
+    return {
+        "list_runs": mcp_tools.list_runs,
+        "get_run_summary": mcp_tools.get_run_summary,
+        "run_evaluation": mcp_tools.run_evaluation,
+        "analyze_compare": mcp_tools.analyze_compare,
+        "get_artifacts": mcp_tools.get_artifacts,
+    }
+
+
+def _allowed_tools(settings: Settings) -> set[str]:
+    if settings.mcp_allowed_tools:
+        return {name.strip() for name in settings.mcp_allowed_tools.split(",") if name.strip()}
+    return set(_tool_registry().keys())
+
+
+def _serialize_result(result: Any) -> Any:
+    if hasattr(result, "model_dump"):
+        return result.model_dump()
+    if hasattr(result, "dict"):
+        return result.dict()
+    try:
+        return asdict(result)
+    except TypeError:
+        return result
+
+
+def _jsonrpc_result(rpc_id: int | str | None, payload: Any) -> dict[str, Any]:
+    return {"jsonrpc": "2.0", "id": rpc_id, "result": payload}
+
+
+def _jsonrpc_error(rpc_id: int | str | None, code: int, message: str) -> dict[str, Any]:
+    return {"jsonrpc": "2.0", "id": rpc_id, "error": {"code": code, "message": message}}
+
+
+@router.post("")
+def handle_mcp_request(
+    request: JsonRpcRequest,
+    settings: Settings = Depends(get_settings),
+    _: None = Depends(_require_mcp_token),
+) -> dict[str, Any]:
+    method = request.method
+    params = request.params or {}
+
+    if method == "initialize":
+        return _jsonrpc_result(
+            request.id,
+            {
+                "protocolVersion": settings.mcp_protocol_version,
+                "serverInfo": {
+                    "name": "evalvault-mcp",
+                    "version": settings.mcp_server_version,
+                },
+                "capabilities": {"tools": {"listChanged": False}},
+            },
+        )
+
+    if method in {"initialized", "notifications/initialized"}:
+        return _jsonrpc_result(request.id, None)
+
+    if method == "tools/list":
+        allowed = _allowed_tools(settings)
+        tools = [tool for tool in mcp_tools.get_tool_specs() if tool.get("name") in allowed]
+        return _jsonrpc_result(request.id, {"tools": tools})
+
+    if method == "tools/call":
+        tool_name = params.get("name")
+        tool_args = params.get("arguments") or {}
+        if not tool_name:
+            return _jsonrpc_error(request.id, -32602, "Missing tool name")
+
+        allowed = _allowed_tools(settings)
+        if tool_name not in allowed:
+            return _jsonrpc_error(request.id, -32601, "Tool not allowed")
+
+        tool_fn = _tool_registry().get(tool_name)
+        if tool_fn is None:
+            return _jsonrpc_error(request.id, -32601, "Tool not found")
+
+        try:
+            result = tool_fn(tool_args)
+        except Exception as exc:
+            return _jsonrpc_error(request.id, -32000, f"Tool execution failed: {exc}")
+
+        payload = _serialize_result(result)
+        return _jsonrpc_result(
+            request.id,
+            {
+                "content": [{"type": "text", "text": json.dumps(payload, ensure_ascii=False)}],
+                "structuredContent": payload,
+            },
+        )
+
+    if method == "ping":
+        return _jsonrpc_result(request.id, {"status": "ok"})
+
+    return _jsonrpc_error(request.id, -32601, "Method not found")

{evalvault-1.66.0.dist-info → evalvault-1.67.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evalvault
-Version: 1.66.0
+Version: 1.67.0
 Summary: RAG evaluation system using Ragas with Phoenix/Langfuse tracing
 Project-URL: Homepage, https://github.com/ntts9990/EvalVault
 Project-URL: Documentation, https://github.com/ntts9990/EvalVault#readme

{evalvault-1.66.0.dist-info → evalvault-1.67.0.dist-info}/RECORD

@@ -6,13 +6,14 @@ evalvault/adapters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 evalvault/adapters/inbound/__init__.py,sha256=SG1svel1PwqetnqVpKFLSv612_WwGwLTbFpYgwk6FMw,166
 evalvault/adapters/inbound/api/__init__.py,sha256=LeVVttCA3tLKoHA2PO4z3y8VkfVcf3Bq8CZSzo91lf4,34
 evalvault/adapters/inbound/api/adapter.py,sha256=HgWSYyUxvJPlaSG158WVzpPckpPCYV9Ec3CWN8rLFdI,69118
-evalvault/adapters/inbound/api/main.py,sha256=skYtmDngdOBryyLXQpNGlSd2Te6RF6GtfIwcMACPHFU,7068
+evalvault/adapters/inbound/api/main.py,sha256=RPcstctf_mFH9TPUhld6plA0104Kb6Iccb6Cu26oFR8,7271
 evalvault/adapters/inbound/api/routers/__init__.py,sha256=q07_YF9TnBl68bqcRCvhPU4-zRTyvmPoHVehwO6W7QM,19
 evalvault/adapters/inbound/api/routers/benchmark.py,sha256=yevntbZcNtMvbVODsITUBgR1Ka4pdFQrXBJJ4K4Jyr4,4477
-evalvault/adapters/inbound/api/routers/chat.py,sha256=3S6-ljiY1COlDuVDH5yzMJs9SO0EkuosRcJIYScHWvI,18143
+evalvault/adapters/inbound/api/routers/chat.py,sha256=hCA6rWr5GT_gCqu75uCqYwy2gOEUd85mlcc5y-ruFTY,20661
 evalvault/adapters/inbound/api/routers/config.py,sha256=LygN0fVMr8NFtj5zuQXnVFhoafx56Txa98vpwtPa4Jc,4104
 evalvault/adapters/inbound/api/routers/domain.py,sha256=RsR7GIFMjccDN7vpG1uDyk9n1DnCTH18JDGAX7o4Qqc,3648
 evalvault/adapters/inbound/api/routers/knowledge.py,sha256=yb_e7OEPtwldOAzHTGiWe7jShHw2JdpOFnzGPMceRsg,7109
+evalvault/adapters/inbound/api/routers/mcp.py,sha256=yHANV7qIXig-7YSiQgXzSTuabqFStH5yT3URyQGY2W4,4764
 evalvault/adapters/inbound/api/routers/pipeline.py,sha256=8UgQzNFHcuqS61s69mOrPee4OMwfxVdvRWHJ2_qYBF0,17175
 evalvault/adapters/inbound/api/routers/runs.py,sha256=rydOvwWk24QIYafu3XYS3oL_VVCE_jHDmjADhA19T1s,40059
 evalvault/adapters/inbound/cli/__init__.py,sha256=a42flC5NK-VfbdbBrE49IrUL5zAyKdXZYJVM6E3NTE0,675
@@ -338,8 +339,8 @@ evalvault/reports/__init__.py,sha256=Bb1X4871msAN8I6PM6nKGED3psPwZt88hXZBAOdH06Y
 evalvault/reports/release_notes.py,sha256=pZj0PBFT-4F_Ty-Kv5P69BuoOnmTCn4kznDcORFJd0w,4011
 evalvault/scripts/__init__.py,sha256=NwEeIFQbkX4ml2R_PhtIoNtArDSX_suuoymgG_7Kwso,89
 evalvault/scripts/regression_runner.py,sha256=SxZori5BZ8jVQ057Mf5V5FPgIVDccrV5oRONmnhuk8w,8438
-evalvault-1.66.0.dist-info/METADATA,sha256=f6jzeYkN1iuFwYJTcI8r5L52hVNZwACOlQuWYvVz_JY,26159
-evalvault-1.66.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-evalvault-1.66.0.dist-info/entry_points.txt,sha256=Oj9Xc5gYcyUYYNmQfWI8NYGw7nN-3M-h2ipHIMlVn6o,65
-evalvault-1.66.0.dist-info/licenses/LICENSE.md,sha256=3RNWY4jjtrQ_yYa-D-7I3XO12Ti7YzxsLV_dpykujvo,11358
-evalvault-1.66.0.dist-info/RECORD,,
+evalvault-1.67.0.dist-info/METADATA,sha256=8KLerbvqXFjDF3iio6CNKbIr5O3YmnnNg5C91hmXqdo,26159
+evalvault-1.67.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+evalvault-1.67.0.dist-info/entry_points.txt,sha256=Oj9Xc5gYcyUYYNmQfWI8NYGw7nN-3M-h2ipHIMlVn6o,65
+evalvault-1.67.0.dist-info/licenses/LICENSE.md,sha256=3RNWY4jjtrQ_yYa-D-7I3XO12Ti7YzxsLV_dpykujvo,11358
+evalvault-1.67.0.dist-info/RECORD,,