union_kb_ingest 1.0.6 → 1.0.8

package/README.md

@@ -10,7 +10,7 @@
 4. 可选调用大模型，把内容整理为项目知识库规范要求的 Markdown 文件。
 5. 默认生成可直接交给知识库项目使用的 `status: active` Markdown 文件。
 
-启用大模型时，工具会把 `prompts/联合运维知识库建立规范.md` 作为格式和质量约束放入提示词，要求模型依据原文语义判断业务场景、模块、角色、标签和风险等级。代码中的启发式生成只作为未启用大模型或调用失败时的兜底，不使用预设业务关键词去指导大模型输出。
+启用大模型时，工具只会读取 `prompts/知识库建立规范.md` 作为格式和质量约束，并由代码按当前片段、辅助上下文和输出 JSON 结构组装生成提示词。模型需依据原文语义判断业务场景、模块、角色、标签和风险等级；代码中的启发式生成只作为未启用大模型时的兜底，不使用预设业务关键词去指导大模型输出。
 
 ## 安装可选依赖
 

package/normalizer.py

@@ -19,6 +19,7 @@ CURRENT_DIR = Path(__file__).resolve().parent
 KB_SPEC_PATH = CURRENT_DIR / "prompts" / "知识库建立规范.md"
 TOOLS_PATH = CURRENT_DIR / "input" / "function" / "tools.json"
 LLM_MAX_RETRIES = 10
+COVERAGE_MAX_RETRIES = 3
 
 
 @dataclass(frozen=True)
@@ -59,6 +60,7 @@ def _normalize_with_llm(block: ParsedBlock, status: str) -> List[KnowledgeItem]:
     client = client_cls(api_key=config.api_key, base_url=config.base_url)
 
     compact_retry = False
+    coverage_retry_count = 0
     coverage_retry_feedback = ""
     json_retry_feedback = ""
     for attempt in range(1, LLM_MAX_RETRIES + 1):
@@ -150,18 +152,29 @@ def _normalize_with_llm(block: ParsedBlock, status: str) -> List[KnowledgeItem]:
         if items:
             coverage_issues = _source_fact_coverage_issues(block, items)
             if coverage_issues:
+                high_relevance_issues = _high_relevance_coverage_issues(
+                    client, config, block, items, coverage_issues
+                )
+                if not high_relevance_issues:
+                    print(
+                        "llm coverage warning ignored: "
+                        "no highly relevant missing facts after relevance review"
+                    )
+                    return items
                 print(
                     "llm coverage failed: "
                     f"missing_facts={len(coverage_issues)} "
-                    f"preview={_preview('；'.join(coverage_issues[:3]))}"
+                    f"high_relevance_missing_facts={len(high_relevance_issues)} "
+                    f"preview={_preview('；'.join(high_relevance_issues[:3]))}"
                 )
-                if attempt >= LLM_MAX_RETRIES:
+                if coverage_retry_count >= COVERAGE_MAX_RETRIES or attempt >= LLM_MAX_RETRIES:
                     print(
                         "WARNING: source fact coverage failed after "
-                        f"{LLM_MAX_RETRIES} attempts; releasing draft for manual review"
+                        f"{coverage_retry_count} coverage retries; releasing draft for manual review"
                     )
-                    return _items_with_coverage_warning(items, block, coverage_issues)
-                coverage_retry_feedback = _coverage_retry_prompt(block, coverage_issues, items)
+                    return _items_with_coverage_warning(items, block, high_relevance_issues)
+                coverage_retry_count += 1
+                coverage_retry_feedback = _coverage_retry_prompt(block, high_relevance_issues, items)
                 time.sleep(min(2 ** (attempt - 1), 30))
                 continue
             return items
@@ -199,6 +212,167 @@ def _compact_retry_prompt(base_prompt: str) -> str:
     )
 
 
+def _high_relevance_coverage_issues(
+    client,
+    config,
+    block: ParsedBlock,
+    items: List[KnowledgeItem],
+    missing_facts: List[str],
+) -> List[str]:
+    """让 LLM 判断缺失事实是否与当前条目高度相关。"""
+    if not missing_facts:
+        return []
+    prompt = _coverage_relevance_prompt(block, items, missing_facts)
+    try:
+        response = _create_zhipu_completion(client, config, prompt)
+        content = _extract_response_content(response)
+        parsed = _extract_json_with_diagnostics(content).value
+        high_relevance = _high_relevance_facts_from_analysis(parsed, missing_facts)
+        if high_relevance is not None:
+            print(
+                "llm coverage relevance: "
+                f"missing_facts={len(missing_facts)} high_relevance={len(high_relevance)}"
+            )
+            return high_relevance
+    except Exception as exc:
+        print(f"llm coverage relevance failed: {type(exc).__name__} detail={exc}")
+
+    fallback = _fallback_high_relevance_coverage_issues(block, items, missing_facts)
+    print(
+        "llm coverage relevance fallback: "
+        f"missing_facts={len(missing_facts)} high_relevance={len(fallback)}"
+    )
+    return fallback
+
+
+def _coverage_relevance_prompt(
+    block: ParsedBlock,
+    items: List[KnowledgeItem],
+    missing_facts: List[str],
+) -> str:
+    """构造缺失事实相关性判定提示。"""
+    fact_lines = "\n".join(f"- {fact}" for fact in missing_facts[:20])
+    current_items = "\n\n".join(
+        f"标题：{item.title}\n核心正文：{_core_sections_for_coverage(item.body)}"
+        for item in items
+    )
+    return f"""
+请判断以下“覆盖校验缺失事实”是否与当前知识条目的主题极高相关。
+
+判定规则：
+1. 只有缺失事实是回答当前条目标题或核心正文所必须保留的定义、规则、阈值、条件、主体、简称、例外或限制时，才标记为“极高”。
+2. 来源文件标题、章节标题、目录项、上级主题名称、页眉页脚、纯标签、仅用于定位的小标题，通常不是“极高”，除非它本身就是当前条目要解释的完整定义或规则。
+3. 辅助上下文只用于理解位置和主题，不要把辅助上下文中独有的信息作为缺失事实依据。
+4. 只能返回 JSON object，不要 Markdown 或解释文字。
+
+返回格式：
+{{
+  "facts": [
+    {{"fact": "必须原样复制待判断事实", "relevance": "极高|一般|低", "reason": "一句话原因"}}
+  ]
+}}
+
+来源文档：{block.source_doc}
+来源章节：{block.source_section or "全文"}
+
+当前来源原文片段：
+{_preview(block.content)[:4000] or "无"}
+
+辅助上下文：
+{_preview(block.context)[:2000] or "无"}
+
+当前已生成条目：
+{_preview(current_items)[:4000] or "无"}
+
+待判断事实：
+{fact_lines}
+""".strip()
+
+
+def _high_relevance_facts_from_analysis(parsed, missing_facts: List[str]):
+    """从相关性判定 JSON 中提取极高相关事实。"""
+    if not isinstance(parsed, dict):
+        return None
+    raw_facts = parsed.get("facts")
+    if raw_facts is None and isinstance(parsed.get("results"), list):
+        raw_facts = parsed.get("results")
+    if raw_facts is None and isinstance(parsed.get("items"), list):
+        raw_facts = parsed.get("items")
+    if not isinstance(raw_facts, list):
+        return None
+
+    missing_by_norm = {_coverage_text(fact): fact for fact in missing_facts}
+    selected: List[str] = []
+    for raw in raw_facts:
+        if not isinstance(raw, dict):
+            continue
+        relevance = str(raw.get("relevance") or raw.get("关联度") or "").strip().lower()
+        if not ("极高" in relevance or "high" in relevance):
+            continue
+        fact = str(raw.get("fact") or raw.get("事实") or raw.get("text") or "").strip()
+        matched = _match_missing_fact(fact, missing_by_norm)
+        if matched and matched not in selected:
+            selected.append(matched)
+    return selected
+
+
+def _match_missing_fact(fact: str, missing_by_norm: Dict[str, str]) -> str:
+    """把模型返回事实匹配回原始缺失事实。"""
+    fact_norm = _coverage_text(fact)
+    if not fact_norm:
+        return ""
+    if fact_norm in missing_by_norm:
+        return missing_by_norm[fact_norm]
+    for missing_norm, missing in missing_by_norm.items():
+        if fact_norm in missing_norm or missing_norm in fact_norm:
+            return missing
+    return ""
+
+
+def _fallback_high_relevance_coverage_issues(
+    block: ParsedBlock,
+    items: List[KnowledgeItem],
+    missing_facts: List[str],
+) -> List[str]:
+    """相关性判定失败时的保守兜底，过滤明显结构性标题。"""
+    return [
+        fact for fact in missing_facts
+        if not _looks_like_structural_missing_fact(block, items, fact)
+    ]
+
+
+def _looks_like_structural_missing_fact(
+    block: ParsedBlock,
+    items: List[KnowledgeItem],
+    fact: str,
+) -> bool:
+    """判断缺失事实是否只是标题、章节或定位信息。"""
+    fact_norm = _coverage_text(fact)
+    if not fact_norm:
+        return True
+    candidates = [
+        block.source_doc,
+        Path(block.source_doc).stem,
+        block.source_section,
+        block.category,
+        block.subcategory,
+        block.source_doc_description,
+        block.subcategory_description,
+        *block.category_path,
+        *block.related_categories,
+        *(item.title for item in items),
+    ]
+    candidate_norms = {_coverage_text(value) for value in candidates if value}
+    if fact_norm in candidate_norms:
+        return True
+    if len(fact_norm) <= 30 and not re.search(
+        r"是|为|指|称|简称|英文|应|需|必须|不得|禁止|超过|低于|大于|小于|不少于|不超过|\d",
+        fact,
+    ):
+        return True
+    return False
+
+
 def _coverage_retry_prompt(
     block: ParsedBlock,
     missing_facts: List[str],
@@ -279,22 +453,76 @@ def _get_zhipu_client_class():
 
 def _extract_response_content(response) -> str:
     """从模型响应中提取正文内容。"""
-    if isinstance(response, dict):
-        choices = response.get("choices") or []
-        if not choices:
-            return ""
-        message = choices[0].get("message") if isinstance(choices[0], dict) else None
-        return str((message or {}).get("content") or "")
-
-    choices = getattr(response, "choices", None) or []
-    if not choices:
+    message = _first_message(response)
+    if message is None:
         return ""
-    message = getattr(choices[0], "message", None)
-    if message is None and isinstance(choices[0], dict):
-        message = choices[0].get("message")
+
     if isinstance(message, dict):
-        return str(message.get("content") or "")
-    return str(getattr(message, "content", "") or "")
+        content = _stringify_message_content(message.get("content"))
+        if content:
+            return content
+        content = _extract_tool_call_content(message.get("function_call"))
+        if content:
+            return content
+        content = _extract_tool_call_content(message.get("tool_calls"))
+        if content:
+            return content
+        return _stringify_message_content(message.get("reasoning_content"))
+
+    content = _stringify_message_content(getattr(message, "content", ""))
+    if content:
+        return content
+    content = _extract_tool_call_content(getattr(message, "function_call", None))
+    if content:
+        return content
+    content = _extract_tool_call_content(getattr(message, "tool_calls", None))
+    if content:
+        return content
+    return _stringify_message_content(getattr(message, "reasoning_content", ""))
+
+
+def _stringify_message_content(content) -> str:
+    """兼容不同 SDK 返回的纯文本、分段文本和结构化 content。"""
+    if content is None:
+        return ""
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        parts = [_stringify_message_content(part) for part in content]
+        return "\n".join(part for part in parts if part)
+    if isinstance(content, dict):
+        for key in ("text", "content", "output_text", "json", "arguments"):
+            value = content.get(key)
+            text = _stringify_message_content(value)
+            if text:
+                return text
+        try:
+            return json.dumps(content, ensure_ascii=False)
+        except TypeError:
+            return str(content)
+
+    for attr in ("text", "content", "output_text"):
+        value = getattr(content, attr, None)
+        text = _stringify_message_content(value)
+        if text:
+            return text
+    return str(content)
+
+
+def _extract_tool_call_content(tool_calls) -> str:
+    """从工具/函数调用参数里兜底提取 JSON 文本。"""
+    if not tool_calls:
+        return ""
+    calls = tool_calls if isinstance(tool_calls, list) else [tool_calls]
+    for call in calls:
+        function = call.get("function") if isinstance(call, dict) else getattr(call, "function", None)
+        if function is None:
+            function = call
+        arguments = function.get("arguments") if isinstance(function, dict) else getattr(function, "arguments", None)
+        text = _stringify_message_content(arguments)
+        if text:
+            return text
+    return ""
 
 
 def _extract_reasoning_content(response) -> str:
@@ -351,7 +579,19 @@ def _coerce_raw_items(parsed):
         if isinstance(items, list):
             return items
 
-        for key in ("knowledge_items", "records", "data", "result", "results"):
+        for key in (
+            "knowledge_items",
+            "records",
+            "data",
+            "payload",
+            "output",
+            "response",
+            "answer",
+            "content",
+            "message",
+            "result",
+            "results",
+        ):
             value = parsed.get(key)
             if isinstance(value, list):
                 print(f"llm parse notice: using non-standard list field '{key}' as items")
@@ -361,6 +601,13 @@ def _coerce_raw_items(parsed):
                 if isinstance(nested, list):
                     print(f"llm parse notice: using nested field '{key}' as items")
                     return nested
+            if isinstance(value, str) and value.strip():
+                nested = _extract_json_with_diagnostics(value)
+                if nested.value is not None:
+                    nested_items = _coerce_raw_items(nested.value)
+                    if isinstance(nested_items, list):
+                        print(f"llm parse notice: parsed JSON string field '{key}' as items")
+                        return nested_items
 
         if _looks_like_single_item(parsed):
             print("llm parse notice: wrapping single item object as items[0]")
@@ -418,7 +665,7 @@ def _build_prompt(block: ParsedBlock, status: str) -> str:
 要求：
 1. 严格参照《知识库建立规范》的元数据字段、正文 5 节结构、内容切分原则和质量校验要求生成。
 2. 只依据原文理解知识点、对象、模块、角色、标签和风险等级，不要依据示例或常见关键词进行套写。
-3. 如果一个片段包含多个独立定义、规则、流程、指标、接口或评价标准，请拆成多个 items。
+3. 如果一个片段包含多个独立定义、规则、流程、指标、接口或判定标准，请拆成多个 items。
 4. 每个 item 必须可独立检索、独立回答，颗粒度控制在 800 到 1500 中文字符左右；复杂表格可适当放宽。
 5. 不要编造来源、阈值、角色、日期、版本；原文没有的信息留空、空数组或使用规范允许的通用值。
 6. 涉及表格、阈值、比较符、单位、持续时间、笔数、适用对象时必须保留原始逻辑。
@@ -447,7 +694,7 @@ doc_type 只能取：
   "items": [
     {{
       "title": "",
-      "doc_type": "scenario",
+      "doc_type": "biz",
       "category": "",
       "subcategory": "",
       "related_items": [],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "union_kb_ingest",
-  "version": "1.0.6",
+  "version": "1.0.8",
   "description": "Offline knowledge-base ingest helper for PDF, Word, Markdown and TXT documents.",
   "bin": {
     "union_kb_ingest": "bin/union_kb_ingest"

package/prompts//347/237/245/350/257/206/345/272/223/345/273/272/347/253/213/350/247/204/350/214/203.md

@@ -14,18 +14,12 @@
 
 ## 2. 条目类型
 
-`doc_type` 用于辅助检索和过滤，必须依据原文内容选择。无法明确归类时使用 `biz`。
+`doc_type` 只做粗粒度标识，用于区分普通知识条目和明确的可调用能力说明。无法明确归类时一律使用 `biz`，不要为了贴合某类业务场景创造或套用细分类型。
 
 | 类型 | 说明 |
 | --- | --- |
-| `biz` | 业务、系统、对象、概念、范围或通用说明 |
-| `scenario` | 场景、状态、事件、问题表现或适用情形 |
-| `sop` | 操作步骤、处理流程、执行要求 |
-| `metric` | 指标定义、统计口径、计算方式、阈值 |
-| `severity` | 分级、定级、优先级或等级判断 |
-| `change` | 变更、发布、评估、通知或回退要求 |
-| `function` | 可调用能力、工具、接口或函数说明 |
-| `evaluation` | 评价对象、评价周期、评价指标、评价结果 |
+| `biz` | 普通知识条目，覆盖概念、对象、范围、规则、流程、指标、阈值、分级、要求等来源事实 |
+| `function` | 来源正文明确描述可调用工具、接口、函数名称、入参或出参时使用 |
 
 ## 3. 单篇知识文档格式
 
@@ -101,7 +95,7 @@ status: "active"
 | --- | --- |
 | `low` | 解释、查询、制度说明、定义说明 |
 | `medium` | 需要判断条件、组合信息或给出建议 |
-| `high` | 涉及生产动作、变更动作、降级、暂停、回退等高影响建议 |
+| `high` | 涉及高影响动作、降级、暂停、回退等需谨慎确认的建议 |
 | `critical` | 涉及不可逆动作、敏感操作或必须人工审批的内容 |
 
 ## 5. 正文内容规范
@@ -152,7 +146,7 @@ status: "active"
 2. 一组紧密相关的规则或条件。
 3. 一个流程或操作要求。
 4. 一个指标口径或阈值表。
-5. 一个分级或评价标准。
+5. 一组等级、优先级或判定标准。
 6. 一个接口、函数或工具说明。
 
 不要为了凑栏目把无关内容合并，也不要把整份制度原文作为一个向量文档直接入库。

package/schemas.py

@@ -7,13 +7,7 @@ from typing import Dict, List
 
 DOC_TYPES = {
     "biz",
-    "scenario",
-    "sop",
-    "metric",
-    "severity",
-    "change",
     "function",
-    "evaluation",
 }
 
 

package/prompts/generate_kb_items.md

@@ -1,27 +0,0 @@
-你是业务知识库整理助手。
-
-请基于输入原文生成标准知识库条目，并严格参照 `prompts/知识库建立规范.md`。必须遵守：
-
-1. 只依据原文，不编造阈值、角色、日期、版本。
-2. 如果一个片段包含多个独立定义、规则、流程、指标、接口或评价标准，拆成多个 items。
-3. 每个 item 需要可独立检索、独立回答。
-4. 保留表格、阈值、比较符、单位、持续时间和适用对象。
-5. 输出严格 JSON，不要 Markdown 代码围栏。
-6. 不要依据预设业务关键词套写业务模块、角色、标签或风险等级，应根据原文语义判断；原文缺失时使用空数组或规范允许的默认值。
-
-输出格式：
-
-{
-  "items": [
-    {
-      "title": "",
-      "doc_type": "scenario",
-      "business_modules": [],
-      "source_version": "",
-      "risk_level": "low",
-      "applicable_roles": [],
-      "tags": [],
-      "body": "# 标题\n\n## 1. 核心内容\n\n...\n\n## 2. 适用边界\n\n...\n\n## 3. 使用要求\n\n...\n\n## 4. 关联能力\n\n...\n\n## 5. 来源依据\n\n..."
-    }
-  ]
-}