union-app-chat-stream 1.0.3

package/app/service/rag_service.py

@@ -0,0 +1,866 @@
+import hashlib
+import json
+import re
+
+try:
+    import pysqlite3
+    import sys
+    sys.modules["sqlite3"] = pysqlite3
+except ImportError:
+    pass
+
+import sqlite3
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+import yaml
+
+from zai import ZhipuAiClient
+
+from loguru import logger
+
+
+def _preview(text: str, limit: int = 300) -> str:
+    return str(text).replace("\n", " ")[:limit]
+
+
+class RagService:
+    """轻量 RAG 服务：加载 Markdown 知识库，写入 Chroma，并按问题检索。"""
+
+    _TOKEN_SPLIT_RE = re.compile(r"[\s,，、;；/／|｜（）()《》【】\\-]+")
+    _CODE_PATTERNS = [
+        re.compile(r"(?<![A-Za-z0-9])[a-zA-Z]+[0-9]+(?:\.[0-9A-Za-z]+)+"),
+        re.compile(r"(?<![A-Za-z0-9])[a-zA-Z]+[0-9]{2,}"),
+        re.compile(r"(?<![A-Za-z0-9])[0-9]{6}-[a-zA-Z0-9-]+-v[0-9]+"),
+        re.compile(r"(?<![A-Za-z0-9])[a-zA-Z]+-offline-[a-fA-F0-9]{8,}-v[0-9]+"),
+        re.compile(r"(?<![A-Za-z0-9])[A-Z]\.[0-9]+(?:\.[0-9]+)?"),
+        re.compile(r"(?<![A-Za-z0-9])[a-z][A-Za-z0-9]*[A-Z][A-Za-z0-9]*"),
+    ]
+
+    def __init__(self, config):
+        self._config = config
+        self._enabled = config["RAG_ENABLED"]
+        self._top_k = self._positive_int(config["RAG_TOP_K"])
+        self._collection = None
+        self._root = Path(__file__).resolve().parents[2]
+        self._persist_dir = self._root / config["RAG_PERSIST_DIR"]
+        self._knowledge_dir = self._root / config["RAG_KNOWLEDGE_DIR"]
+        self._collection_name = config["RAG_COLLECTION"]
+        self._rebuild_on_startup = config["RAG_REBUILD_ON_STARTUP"]
+        self._embedding_model = config["RAG_EMBEDDING_MODEL"]
+        self._embedding_max_chars = self._positive_int(config["RAG_EMBEDDING_MAX_CHARS"])
+        self._embedding_batch_size = self._positive_int(config["RAG_EMBEDDING_BATCH_SIZE"])
+        self._semantic_candidate_k = self._positive_int(config["RAG_SEMANTIC_CANDIDATE_K"])
+        self._context_k = self._positive_int(config["RAG_CONTEXT_K"])
+        self._exact_context_k = self._positive_int(config["RAG_EXACT_CONTEXT_K"])
+        self._exact_per_file_context_k = self._positive_int(config["RAG_EXACT_PER_FILE_CONTEXT_K"])
+        self._per_file_context_k = self._positive_int(config["RAG_PER_FILE_CONTEXT_K"])
+        self._chunk_size = self._positive_int(config["RAG_CHUNK_SIZE"])
+        self._chroma = None
+        self._client = ZhipuAiClient(
+            api_key=config["LLM_KEY"],
+            base_url=config["LLM_URL"],
+        )
+        if self._enabled:
+            self._init_collection()
+
+    def search(self, question: str, top_k: Optional[int] = None) -> Tuple[str, List[Dict]]:
+        if not self._ensure_collection():
+            logger.info(f"RAG 未启用或集合不可用，跳过检索。question={_preview(question, 120)}")
+            return "", []
+
+        exact_terms = self._extract_exact_terms(question)
+        exact_hits = self._exact_match_documents(question, exact_terms)
+        expanded_hits = self._expand_neighbors(exact_hits)
+        relation_hits = self._expand_related_items(exact_hits)
+        try:
+            if self._collection.count() == 0:
+                logger.info("RAG 集合为空，开始重建知识库。")
+                self.rebuild()
+            if self._collection.count() == 0:
+                logger.info(f"RAG 重建后仍无可用文档。question={_preview(question, 120)}")
+                return "", []
+            result = self._collection.query(
+                query_embeddings=[self._embed(question)],
+                n_results=self._semantic_candidate_count(),
+                include=["documents", "metadatas", "distances"],
+            )
+        except Exception:
+            logger.exception(f"RAG 检索异常，已降级为空上下文。question={_preview(question, 120)}")
+            result = {}
+        docs = result.get("documents", [[]])[0]
+        metas = result.get("metadatas", [[]])[0]
+        distances = result.get("distances", [[]])[0]
+        ids = result.get("ids", [[]])[0]
+        semantic_hits = self._semantic_hits(question, docs, metas, distances, ids)
+        ranked = self._merge_and_rank(question, exact_terms, exact_hits, expanded_hits, relation_hits, semantic_hits, top_k)
+        metas = [item["metadata"] for item in ranked]
+        sources = [self._source(m) for m in metas]
+        context = self._format_evidence_context(ranked)
+        logger.info(f"RAG 检索完成。question={_preview(question, 120)} hit_count={len(ranked)} exact_terms={exact_terms} sources={sources} context_preview={_preview(context)}")
+        return context, sources
+
+    def knowledge_search(self, query: str, top_k: Optional[Any] = None) -> Tuple[Dict[str, Any], str]:
+        query = str(query or "").strip()
+        if not query:
+            return {"context": "", "sources": [], "related_tools": []}, "query_empty"
+
+        context, sources = self.search(query, top_k=self._parse_optional_top_k(top_k))
+        return {
+            "query": query,
+            "context": context,
+            "sources": sources,
+            "related_tools": self._extract_related_tools(sources),
+        }, "success"
+
+    def _ensure_collection(self) -> bool:
+        if self._collection:
+            return True
+        if not self._enabled:
+            return False
+        self._init_collection()
+        return self._collection is not None
+
+    @staticmethod
+    def _positive_int(value) -> int:
+        parsed = int(value)
+        if parsed < 1:
+            raise ValueError(f"配置值必须为正整数: {value}")
+        return parsed
+
+    def _parse_optional_top_k(self, value: Optional[Any]) -> Optional[int]:
+        if value in (None, ""):
+            return None
+        parsed = self._positive_int(value)
+        return min(parsed, self._context_limit())
+
+    @staticmethod
+    def _extract_related_tools(sources: List[Dict]) -> List[Dict[str, str]]:
+        related_tools: Dict[str, Dict[str, str]] = {}
+        for source in sources:
+            raw = source.get("related_items", "")
+            if not raw:
+                continue
+            text = raw if isinstance(raw, str) else json.dumps(raw, ensure_ascii=False)
+            for name in sorted(set(re.findall(r"\b[a-zA-Z][a-zA-Z0-9_]{2,}\b", text))):
+                if "_" not in name:
+                    continue
+                related_tools.setdefault(name, {
+                    "name": name,
+                    "reason": "知识库关联能力提示；是否可执行以当前 tool definition 为准。",
+                })
+        return list(related_tools.values())
+
+    def _init_collection(self):
+        try:
+            import chromadb
+        except ImportError:
+            logger.warning("未安装 chromadb，RAG 检索不可用。")
+            return
+
+        self._chroma = chromadb.PersistentClient(path=str(self._persist_dir))
+        self._collection = self._chroma.get_or_create_collection(self._collection_name)
+        try:
+            if self._rebuild_on_startup:
+                self.rebuild()
+        except Exception:
+            logger.exception("RAG 初始化重建失败，已关闭当前集合。")
+            self._collection = None
+
+    def rebuild(self) -> int:
+        docs = self._load_documents()
+        if not docs:
+            logger.info("RAG 未加载到知识库文档，跳过重建。")
+            return 0
+        embeddings = []
+        logger.info(f"RAG 开始生成文档向量。doc_chunks={len(docs)}")
+        for start in range(0, len(docs), self._embedding_batch_size):
+            batch_docs = docs[start:start + self._embedding_batch_size]
+            batch_end = start + len(batch_docs)
+            try:
+                logger.info(
+                    f"RAG 批量生成文档向量中。progress={start + 1}-{batch_end}/{len(docs)} "
+                    f"batch_size={len(batch_docs)} "
+                    f"first_file={batch_docs[0]['metadata'].get('file_path', '')} "
+                    f"first_chunk_index={batch_docs[0]['metadata'].get('chunk_index', '')} "
+                    f"first_title={batch_docs[0]['metadata'].get('title', '')}"
+                )
+                embeddings.extend(self._embed_batch([doc["content"] for doc in batch_docs]))
+                logger.info(
+                    f"RAG 批量生成文档向量完成。progress={batch_end}/{len(docs)} "
+                    f"batch_size={len(batch_docs)}"
+                )
+            except Exception:
+                first_doc = batch_docs[0]
+                first_metadata = first_doc["metadata"]
+                logger.exception(
+                    f"RAG 批量文档向量生成失败。progress={start + 1}-{batch_end}/{len(docs)} "
+                    f"first_file={first_metadata.get('file_path', '')} "
+                    f"first_chunk_index={first_metadata.get('chunk_index', '')} "
+                    f"first_title={first_metadata.get('title', '')} "
+                    f"first_content_preview={_preview(first_doc['content'], 120)}"
+                )
+                raise
+        if len(embeddings) != len(docs):
+            raise RuntimeError(f"RAG 文档向量数量不匹配。docs={len(docs)} embeddings={len(embeddings)}")
+        self._collection.upsert(
+            ids=[d["id"] for d in docs],
+            documents=[d["content"] for d in docs],
+            metadatas=[d["metadata"] for d in docs],
+            embeddings=embeddings,
+        )
+        logger.info(f"RAG 知识库重建完成。doc_chunks={len(docs)}")
+        return len(docs)
+
+    def force_rebuild(self) -> Dict:
+        if not self._enabled:
+            return {
+                "enabled": False,
+                "status": "disabled",
+                "message": "RAG 未启用，未执行重建。",
+            }
+        if not self._ensure_collection():
+            return {
+                "enabled": True,
+                "status": "unavailable",
+                "message": "RAG 集合不可用，无法重建。",
+            }
+
+        logger.info(f"RAG 开始强制重建，将删除并重建集合。collection={self._collection_name}")
+        self._chroma.delete_collection(self._collection_name)
+        self._collection = self._chroma.get_or_create_collection(self._collection_name)
+        rebuilt_chunks = self.rebuild()
+        status = self.check()
+        status.update({
+            "status": "rebuilt",
+            "rebuilt_chunks": rebuilt_chunks,
+        })
+        return status
+
+    def check(self) -> Dict:
+        docs = self._load_documents()
+        expected_ids = {d["id"] for d in docs}
+        source_files = [str(path) for path in self._knowledge_dir.rglob("*.md")]
+        active_files = {str(d["metadata"].get("file_path", "")) for d in docs}
+
+        database = self._database_info()
+        result = {
+            "enabled": self._enabled,
+            "status": "unknown",
+            "collection": self._collection_name,
+            "is_synced": False,
+            "needs_rebuild": True,
+            "last_build_at": database.get("last_embedding_created_at"),
+            "database_updated_at": database.get("modified_at"),
+            "item_counts": {
+                "source_files": len(source_files),
+                "active_source_files": len(active_files),
+                "expected_chunks": len(docs),
+                "vector_chunks": 0,
+                "missing_chunks": len(expected_ids),
+                "stale_chunks": 0,
+            },
+            "config": {
+                "embedding_model": self._embedding_model,
+                "top_k": self._top_k,
+                "chunk_size": self._chunk_size,
+            },
+        }
+
+        if not self._ensure_collection():
+            result.update({
+                "status": "disabled" if not self._enabled else "unavailable",
+                "message": "RAG 未启用或集合不可用。",
+            })
+            return result
+
+        vector_count = self._collection.count()
+        result["item_counts"]["vector_chunks"] = vector_count
+        stored_ids = set()
+        if vector_count:
+            stored = self._collection.get(limit=vector_count, include=[])
+            stored_ids = set(stored.get("ids", []))
+
+        missing_ids = expected_ids - stored_ids
+        stale_ids = stored_ids - expected_ids
+        is_synced = not missing_ids and not stale_ids and vector_count == len(expected_ids)
+        result.update({
+            "status": "ready" if vector_count else "empty",
+            "is_synced": is_synced,
+            "needs_rebuild": not is_synced,
+        })
+        result["item_counts"].update({
+            "missing_chunks": len(missing_ids),
+            "stale_chunks": len(stale_ids),
+        })
+        return result
+
+    def _database_info(self) -> Dict:
+        db_path = self._persist_dir / "chroma.sqlite3"
+        info = {
+            "path": str(db_path),
+            "exists": db_path.exists(),
+        }
+        if not db_path.exists():
+            return info
+
+        stat = db_path.stat()
+        info.update({
+            "size_bytes": stat.st_size,
+            "modified_at": datetime.fromtimestamp(stat.st_mtime).isoformat(timespec="seconds"),
+        })
+        try:
+            with sqlite3.connect(str(db_path)) as conn:
+                row = conn.execute("select min(created_at), max(created_at) from embeddings").fetchone()
+            info.update({
+                "first_embedding_created_at": row[0] if row else None,
+                "last_embedding_created_at": row[1] if row else None,
+            })
+        except sqlite3.Error:
+            logger.exception(f"读取 Chroma SQLite 元信息失败。path={db_path}")
+        return info
+
+    def _load_documents(self) -> List[Dict]:
+        docs = []
+        for path in self._knowledge_dir.rglob("*.md"):
+            metadata, body = self._read_markdown(path)
+            if metadata.get("status", "active") != "active":
+                continue
+            for idx, chunk in enumerate(self._split(body)):
+                item_meta = self._clean_metadata({**metadata, "file_path": str(path), "chunk_index": idx})
+                searchable = self._searchable_content(item_meta, chunk)
+                docs.append({
+                    "id": self._chunk_id(path, idx, chunk),
+                    "content": searchable,
+                    "metadata": item_meta,
+                })
+        return docs
+
+    def _semantic_candidate_count(self) -> int:
+        return self._semantic_candidate_k
+
+    def _context_limit(self) -> int:
+        return self._context_k
+
+    def _semantic_hits(
+            self,
+            question: str,
+            docs: List[str],
+            metas: List[Dict],
+            distances: List[float],
+            ids: List[str],
+    ) -> List[Dict]:
+        hits = []
+        for idx, (doc, meta) in enumerate(zip(docs, metas)):
+            distance = distances[idx] if idx < len(distances) else 1.0
+            hits.append({
+                "id": ids[idx] if idx < len(ids) else self._chunk_id_from_metadata(meta, doc),
+                "document": self._strip_search_prefix(doc),
+                "metadata": meta,
+                "distance": float(distance),
+                "score": (1.0 - float(distance)) + self._metadata_match_score(question, [], meta),
+                "match_type": "semantic",
+                "match_terms": [],
+                "confidence": "medium",
+                "usage_hint": "语义相似证据，可用于补充业务背景、相邻规范或相似场景。",
+            })
+        return hits
+
+    @staticmethod
+    def _chunk_id_from_metadata(metadata: Dict, document: str) -> str:
+        path = Path(str(metadata.get("file_path", "")))
+        idx = metadata.get("chunk_index", "")
+        return RagService._chunk_id(path, idx, document)
+
+    def _extract_exact_terms(self, question: str) -> List[str]:
+        terms = []
+        for pattern in self._CODE_PATTERNS:
+            terms.extend(match.group(0) for match in pattern.finditer(question))
+
+        for token in self._TOKEN_SPLIT_RE.split(question):
+            token = token.strip()
+            if len(token) < 3:
+                continue
+            if re.fullmatch(r"[A-Za-z]+", token) and len(token) < 6:
+                continue
+            if re.search(r"[A-Za-z0-9]", token) and not self._is_generic_ascii_token(token):
+                terms.append(token)
+
+        normalized = []
+        seen = set()
+        for term in terms:
+            term = term.strip("：:，,。.;；()（）[]【】")
+            key = term.lower()
+            if len(term) >= 3 and key not in seen:
+                seen.add(key)
+                normalized.append(term)
+        return normalized
+
+    @staticmethod
+    def _is_generic_ascii_token(token: str) -> bool:
+        return token.lower() in {
+            "json",
+            "http",
+            "https",
+            "api",
+            "xml",
+            "true",
+            "false",
+            "none",
+            "null",
+            "biz",
+            "offline",
+        }
+
+    def _exact_match_documents(self, question: str, exact_terms: List[str]) -> List[Dict]:
+        hits = []
+        for doc in self._load_documents():
+            metadata = doc["metadata"]
+            document = self._strip_search_prefix(doc["content"])
+            metadata_text = self._metadata_text(metadata)
+            metadata_terms = self._matched_terms(metadata_text, exact_terms)
+            body_terms = self._matched_terms(document, exact_terms)
+            name_terms = self._matched_name_terms(question, metadata)
+            matched_terms = sorted(set(metadata_terms + body_terms + name_terms), key=str.lower)
+            if not matched_terms:
+                continue
+            score = (
+                    3.0
+                    + self._metadata_match_score(question, matched_terms, metadata)
+                    + min(len(metadata_terms) * 0.5, 2.0)
+                    + min(len(body_terms) * 1.5, 4.5)
+                    + min(len(name_terms) * 1.2, 3.6)
+            )
+            hits.append({
+                "id": doc["id"],
+                "document": document,
+                "metadata": metadata,
+                "distance": None,
+                "score": score,
+                "match_type": "exact",
+                "match_terms": matched_terms,
+                "body_match_terms": body_terms,
+                "name_match_terms": name_terms,
+                "confidence": "high",
+                "usage_hint": "高置信精确命中，优先用于确定具体对象、字段、格式、编号或阈值。",
+            })
+        return self._prefer_direct_exact_hits(hits)
+
+    @staticmethod
+    def _prefer_direct_exact_hits(hits: List[Dict]) -> List[Dict]:
+        direct_by_file = {}
+        for hit in hits:
+            file_path = hit["metadata"].get("file_path", "")
+            if hit.get("body_match_terms") or hit.get("name_match_terms"):
+                direct_by_file[file_path] = True
+
+        filtered = [
+            hit
+            for hit in hits
+            if hit.get("body_match_terms")
+               or hit.get("name_match_terms")
+               or not direct_by_file.get(hit["metadata"].get("file_path", ""))
+        ]
+        return sorted(filtered, key=lambda item: item["score"], reverse=True)
+
+    @staticmethod
+    def _matched_name_terms(question: str, metadata: Dict) -> List[str]:
+        candidates = [
+            str(metadata.get("title", "")),
+            str(metadata.get("source_section", "")),
+        ]
+        terms = []
+        for candidate in candidates:
+            candidate = candidate.strip()
+            if len(candidate) >= 6 and candidate in question:
+                terms.append(candidate)
+            section_name = re.sub(r"^[A-Z]\.[0-9]+(?:\.[0-9]+)?\s*", "", candidate).strip()
+            if len(section_name) >= 6 and section_name in question:
+                terms.append(section_name)
+        return sorted(set(terms), key=len, reverse=True)
+
+    def _expand_neighbors(self, exact_hits: List[Dict]) -> List[Dict]:
+        if not exact_hits:
+            return []
+
+        docs = self._load_documents()
+        by_location = {
+            (doc["metadata"].get("file_path", ""), int(doc["metadata"].get("chunk_index", 0))): doc
+            for doc in docs
+        }
+        neighbors = []
+        seen = {hit["id"] for hit in exact_hits}
+        for hit in exact_hits[:3]:
+            metadata = hit["metadata"]
+            file_path = metadata.get("file_path", "")
+            try:
+                chunk_index = int(metadata.get("chunk_index", 0))
+            except (TypeError, ValueError):
+                continue
+            for offset in (-1, 1):
+                doc = by_location.get((file_path, chunk_index + offset))
+                if not doc or doc["id"] in seen:
+                    continue
+                seen.add(doc["id"])
+                neighbors.append({
+                    "id": doc["id"],
+                    "document": self._strip_search_prefix(doc["content"]),
+                    "metadata": doc["metadata"],
+                    "distance": None,
+                    "score": hit["score"] - 0.4,
+                    "match_type": "neighbor_context",
+                    "match_terms": hit.get("match_terms", []),
+                    "confidence": "medium",
+                    "usage_hint": "精确命中文档的邻近片段，可用于补充上下文、边界和使用要求。",
+                })
+        return neighbors
+
+    def _expand_related_items(self, exact_hits: List[Dict]) -> List[Dict]:
+        if not exact_hits:
+            return []
+
+        docs = self._load_documents()
+        relation_docs = {}
+        for doc in docs:
+            metadata = doc["metadata"]
+            if int(metadata.get("chunk_index", 0)) != 0:
+                continue
+            category = str(metadata.get("category", "")).strip()
+            for title in (metadata.get("subcategory", ""), metadata.get("title", "")):
+                title = str(title).strip()
+                if title:
+                    relation_docs[(category, title)] = doc
+
+        related = []
+        seen = {hit["id"] for hit in exact_hits}
+        for hit in exact_hits[:3]:
+            metadata = hit["metadata"]
+            source_category = str(metadata.get("category", "")).strip()
+            for item in self._related_items(metadata):
+                target_category = str(item.get("大类标题") or item.get("category") or source_category).strip()
+                target_title = str(item.get("小类标题") or item.get("subcategory") or item.get("title") or "").strip()
+                if not target_title:
+                    continue
+                doc = relation_docs.get((target_category, target_title)) or relation_docs.get(
+                    (source_category, target_title))
+                if not doc:
+                    doc = self._find_related_doc(docs, target_category or source_category, target_title)
+                if not doc or doc["id"] in seen:
+                    continue
+                seen.add(doc["id"])
+                related.append({
+                    "id": doc["id"],
+                    "document": self._strip_search_prefix(doc["content"]),
+                    "metadata": doc["metadata"],
+                    "distance": None,
+                    "score": hit["score"] - 0.2,
+                    "match_type": "relation_context",
+                    "match_terms": hit.get("match_terms", []),
+                    "confidence": "medium",
+                    "usage_hint": "知识小类关系证据，可用于说明同一知识大类下小类之间的关联、边界和相似场景。",
+                })
+        return related
+
+    @staticmethod
+    def _related_items(metadata: Dict) -> List[Dict]:
+        raw = metadata.get("related_items", "")
+        if not raw:
+            return []
+        if isinstance(raw, list):
+            return [item for item in raw if isinstance(item, dict)]
+        if isinstance(raw, str):
+            try:
+                parsed = json.loads(raw)
+            except json.JSONDecodeError:
+                return []
+            return [item for item in parsed if isinstance(item, dict)] if isinstance(parsed, list) else []
+        return []
+
+    @staticmethod
+    def _find_related_doc(docs: List[Dict], category: str, title: str) -> Dict:
+        title_lower = title.lower()
+        for doc in docs:
+            metadata = doc["metadata"]
+            if int(metadata.get("chunk_index", 0)) != 0:
+                continue
+            doc_category = str(metadata.get("category", "")).strip()
+            if category and doc_category and category != doc_category:
+                continue
+            candidates = [
+                str(metadata.get("subcategory", "")).strip(),
+                str(metadata.get("title", "")).strip(),
+                str(metadata.get("source_section", "")).strip(),
+            ]
+            if any(title_lower in candidate.lower() or candidate.lower() in title_lower for candidate in candidates if
+                   candidate):
+                return doc
+        return {}
+
+    def _merge_and_rank(
+            self,
+            question: str,
+            exact_terms: List[str],
+            exact_hits: List[Dict],
+            expanded_hits: List[Dict],
+            relation_hits: List[Dict],
+            semantic_hits: List[Dict],
+            top_k: Optional[int] = None,
+    ) -> List[Dict]:
+        merged = {}
+        for item in [*semantic_hits, *relation_hits, *expanded_hits, *exact_hits]:
+            item = dict(item)
+            searchable = self._evidence_text(item["metadata"], item["document"])
+            matched_terms = self._matched_terms(searchable, exact_terms)
+            item["match_terms"] = sorted(set(item.get("match_terms", []) + matched_terms), key=str.lower)
+            item["score"] = item.get("score", 0.0) + self._metadata_match_score(question, item["match_terms"],
+                                                                                item["metadata"])
+            existing = merged.get(item["id"])
+            if existing is None or self._hit_priority(item) > self._hit_priority(existing):
+                merged[item["id"]] = item
+            elif existing is not None:
+                existing["match_terms"] = sorted(set(existing.get("match_terms", []) + item["match_terms"]),
+                                                 key=str.lower)
+                existing["score"] = max(existing.get("score", 0.0), item.get("score", 0.0))
+
+        ranked = sorted(merged.values(), key=lambda item: self._hit_priority(item), reverse=True)
+        return self._diversify_hits(ranked, top_k or self._context_limit())
+
+    @staticmethod
+    def _hit_priority(item: Dict) -> Tuple[float, int]:
+        type_bonus = {
+            "exact": 3,
+            "neighbor_context": 2,
+            "relation_context": 2,
+            "semantic": 1,
+        }.get(item.get("match_type"), 0)
+        return float(item.get("score", 0.0)), type_bonus
+
+    def _diversify_hits(self, ranked: List[Dict], limit: int) -> List[Dict]:
+        selected = []
+        per_file = {}
+        exact_per_file = {}
+        exact_selected = 0
+        exact_limit = self._exact_context_k
+        exact_file_limit = self._exact_per_file_context_k
+        per_file_limit = self._per_file_context_k
+
+        for item in ranked:
+            file_path = item["metadata"].get("file_path", "")
+            match_type = item.get("match_type")
+            if match_type == "exact" and exact_selected >= exact_limit:
+                continue
+            if match_type == "exact" and exact_per_file.get(file_path, 0) >= exact_file_limit:
+                continue
+            if per_file.get(file_path, 0) >= per_file_limit and match_type != "exact":
+                continue
+            selected.append(item)
+            per_file[file_path] = per_file.get(file_path, 0) + 1
+            if match_type == "exact":
+                exact_selected += 1
+                exact_per_file[file_path] = exact_per_file.get(file_path, 0) + 1
+            if len(selected) >= limit:
+                break
+        return selected
+
+    def _metadata_match_score(self, question: str, matched_terms: List[str], metadata: Dict) -> float:
+        score = 0.0
+        weighted_fields = [
+            ("kb_id", 2.0),
+            ("title", 1.4),
+            ("source_section", 1.2),
+            ("tags", 1.6),
+            ("business_modules", 0.8),
+            ("source_doc", 0.4),
+            ("doc_type", 0.5),
+            ("domain", 0.4),
+            ("category", 0.3),
+            ("category_keywords", 0.3),
+            ("subcategory", 0.7),
+            ("related_items", 0.8),
+            ("related_categories", 0.2),
+            ("relation_notes", 0.2),
+        ]
+        question_lower = question.lower()
+        for field, weight in weighted_fields:
+            text = str(metadata.get(field, ""))
+            text_lower = text.lower()
+            for term in matched_terms:
+                if term.lower() in text_lower:
+                    score += weight
+            for token in self._TOKEN_SPLIT_RE.split(text):
+                token = token.strip()
+                if len(token) >= 2 and token.lower() in question_lower:
+                    score += min(weight, 0.12)
+        return min(score, 8.0)
+
+    @staticmethod
+    def _matched_terms(text: str, terms: List[str]) -> List[str]:
+        lower_text = text.lower()
+        return [term for term in terms if term.lower() in lower_text]
+
+    @staticmethod
+    def _evidence_text(metadata: Dict, document: str) -> str:
+        return f"{RagService._metadata_text(metadata)}\n{document}"
+
+    @staticmethod
+    def _metadata_text(metadata: Dict) -> str:
+        metadata_text = " ".join([
+            str(metadata.get("kb_id", "")),
+            str(metadata.get("title", "")),
+            str(metadata.get("source_doc", "")),
+            str(metadata.get("source_section", "")),
+            str(metadata.get("doc_type", "")),
+            str(metadata.get("domain", "")),
+            str(metadata.get("tags", "")),
+            str(metadata.get("business_modules", "")),
+            str(metadata.get("category", "")),
+            str(metadata.get("category_keywords", "")),
+            str(metadata.get("subcategory", "")),
+            str(metadata.get("related_items", "")),
+            str(metadata.get("related_categories", "")),
+            str(metadata.get("relation_notes", "")),
+            str(metadata.get("file_path", "")),
+        ])
+        return metadata_text
+
+    @staticmethod
+    def _format_evidence_context(ranked: List[Dict]) -> str:
+        group_names = {
+            "exact": "高置信精确命中",
+            "neighbor_context": "精确命中邻近上下文",
+            "relation_context": "知识小类关系上下文",
+            "semantic": "相关语义召回",
+        }
+        groups = []
+        for match_type, group_name in group_names.items():
+            items = [item for item in ranked if item.get("match_type") == match_type]
+            if not items:
+                continue
+            blocks = [f"【{group_name}】"]
+            for item in items:
+                metadata = item["metadata"]
+                terms = "、".join(item.get("match_terms", [])) or "无显式代码词命中"
+                blocks.append(
+                    f"[{len(blocks)}] 标题：{metadata.get('title', '')}\n"
+                    f"匹配类型：{item.get('match_type', '')}\n"
+                    f"置信级别：{item.get('confidence', '')}\n"
+                    f"命中词：{terms}\n"
+                    f"使用建议：{item.get('usage_hint', '')}\n"
+                    f"文档类型：{metadata.get('doc_type', '')}\n"
+                    f"知识领域：{metadata.get('domain', '')}\n"
+                    f"知识大类：{metadata.get('category', '')}\n"
+                    f"关键词：{metadata.get('category_keywords', '')}\n"
+                    f"知识小类：{metadata.get('subcategory', '')}\n"
+                    f"关联信息：{metadata.get('related_items', '')}\n"
+                    f"来源：{metadata.get('source_doc', '')} {metadata.get('source_section', '')}\n"
+                    f"内容：{item.get('document', '')}"
+                )
+            groups.append("\n\n".join(blocks))
+        return "\n\n".join(groups)
+
+    @staticmethod
+    def _searchable_content(metadata: Dict, chunk: str) -> str:
+        prefix = "\n".join([
+            f"知识库ID：{metadata.get('kb_id', '')}",
+            f"标题：{metadata.get('title', '')}",
+            f"来源文档：{metadata.get('source_doc', '')}",
+            f"来源章节：{metadata.get('source_section', '')}",
+            f"文档类型：{metadata.get('doc_type', '')}",
+            f"知识领域：{metadata.get('domain', '')}",
+            f"标签：{metadata.get('tags', '')}",
+            f"业务模块：{metadata.get('business_modules', '')}",
+            f"知识大类：{metadata.get('category', '')}",
+            f"关键词：{metadata.get('category_keywords', '')}",
+            f"知识小类：{metadata.get('subcategory', '')}",
+            f"关联信息：{metadata.get('related_items', '')}",
+        ]).strip()
+        return f"{prefix}\n\n---CONTENT---\n{chunk}" if prefix else chunk
+
+    @staticmethod
+    def _strip_search_prefix(document: str) -> str:
+        marker = "---CONTENT---\n"
+        if marker in document:
+            return document.split(marker, 1)[1]
+        return document
+
+    @staticmethod
+    def _read_markdown(path: Path) -> Tuple[Dict, str]:
+        text = path.read_text(encoding="utf-8")
+        match = re.match(r"^---\n(.*?)\n---\n(.*)$", text, re.S)
+        if not match:
+            return {"title": path.stem, "status": "active"}, text
+        return yaml.safe_load(match.group(1)) or {}, match.group(2).strip()
+
+    def _split(self, text: str) -> List[str]:
+        max_chars = self._chunk_size
+        parts = re.split(r"\n(?=##\s+)", text)
+        chunks = []
+        for part in parts:
+            part = part.strip()
+            if not part:
+                continue
+            if len(part) <= max_chars:
+                chunks.append(part)
+            else:
+                chunks.extend(part[i:i + max_chars] for i in range(0, len(part), max_chars))
+        return chunks
+
+    def _embed(self, text: str) -> List[float]:
+        response = self._client.embeddings.create(
+            model=self._embedding_model,
+            input=text[: self._embedding_max_chars],
+        )
+        item = response.data[0]
+        return item["embedding"] if isinstance(item, dict) else item.embedding
+
+    def _embed_batch(self, texts: List[str]) -> List[List[float]]:
+        response = self._client.embeddings.create(
+            model=self._embedding_model,
+            input=[text[: self._embedding_max_chars] for text in texts],
+        )
+        embeddings = [
+            item["embedding"] if isinstance(item, dict) else item.embedding
+            for item in response.data
+        ]
+        if len(embeddings) != len(texts):
+            raise RuntimeError(f"embedding batch 返回数量不匹配。expected={len(texts)} actual={len(embeddings)}")
+        return embeddings
+
+    @staticmethod
+    def _clean_metadata(metadata: Dict) -> Dict:
+        cleaned = {}
+        for key, value in metadata.items():
+            if isinstance(value, (list, dict)):
+                cleaned[key] = json.dumps(value, ensure_ascii=False)
+            elif value is None:
+                cleaned[key] = ""
+            elif isinstance(value, (str, int, float, bool)):
+                cleaned[key] = value
+            else:
+                cleaned[key] = str(value)
+        return cleaned
+
+    @staticmethod
+    def _chunk_id(path: Path, idx: int, chunk: str) -> str:
+        digest = hashlib.md5(f"{path}:{idx}:{chunk}".encode("utf-8")).hexdigest()
+        return digest
+
+    @staticmethod
+    def _source(metadata: Dict) -> Dict:
+        return {
+            "kb_id": metadata.get("kb_id", ""),
+            "title": metadata.get("title", ""),
+            "doc_type": metadata.get("doc_type", ""),
+            "domain": metadata.get("domain", ""),
+            "category": metadata.get("category", ""),
+            "category_keywords": metadata.get("category_keywords", ""),
+            "source_doc_description": metadata.get("source_doc_description", ""),
+            "subcategory": metadata.get("subcategory", ""),
+            "related_items": metadata.get("related_items", ""),
+            "related_categories": metadata.get("related_categories", ""),
+            "relation_notes": metadata.get("relation_notes", ""),
+            "source_doc": metadata.get("source_doc", ""),
+            "source_section": metadata.get("source_section", ""),
+            "file_path": metadata.get("file_path", ""),
+        }