union_kb_ingest 1.0.10 → 1.0.11

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Simon
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -12,23 +12,11 @@
 
 启用大模型时，工具只会读取 `prompts/知识库建立规范.md` 作为格式和质量约束，并由代码按当前片段、辅助上下文和输出 JSON 结构组装生成提示词。模型需依据原文语义判断业务场景、模块、角色、标签和风险等级；代码中的启发式生成只作为未启用大模型时的兜底，不使用预设业务关键词去指导大模型输出。
 
-## 安装可选依赖
-
-不建议把这些依赖加入项目根 `requirements.txt`。离线机器单独安装即可：
+## 安装依赖
 
 ```bash
 python -m pip install -r requirements.txt
 ```
-
-解析层不使用 OCR，不加载本地视觉/版面模型，也不访问远程模型服务：
-
-- PDF：使用 `docling-parse` 抽取 PDF 内嵌文本和文本行顺序；扫描件或图片型 PDF 不会识别。
-- DOCX：使用 Docling 的 Word 后端转为 Markdown。
-- 旧版 `.doc`：通过 LibreOffice `soffice` 转为 `.docx` 后再解析；不使用 OCR。
-- Markdown / TXT：作为已文本化材料直接读取。
-
-不要安装 `docling` 或 `docling-slim[standard]`，它们会引入 OCR、版面/表格模型、Torch/ONNXRuntime 等重依赖，并可能在运行时下载模型。内网机器建议为离线工具单独准备 Python 3.10+ 环境。
-
 ## 基本用法
 
 把文件放入：
@@ -37,6 +25,8 @@ python -m pip install -r requirements.txt
 input/
 ```
 
+工具调用示例放在 `input/function/tools.yaml` 中，可按业务需要替换。
+
 生成知识库文件：
 
 ```bash
@@ -67,7 +57,7 @@ python ingest.py validate
 
 ## 大模型配置
 
-默认不强制调用大模型，会使用启发式模板生成知识库文件。
+默认不强制调用大模型，但是强烈建议启用大模型分析，会使用启发式模板生成知识库文件。
 
 如果要启用大模型整理，修改 `config/config.yaml`：
 
@@ -75,7 +65,7 @@ python ingest.py validate
 llm:
   enabled: true
   base_url: "https://open.bigmodel.cn/api/paas/v4/"
-  api_key: "your-zhipu-api-key"
+  api_key: ""
   model: "glm-4.7"
   timeout_seconds: 120
   max_tokens: 8192
@@ -87,19 +77,4 @@ draft:
   outline_max_sections: 40
 ```
 
-也可以继续使用环境变量覆盖配置文件：
-
-```bash
-export KB_LLM_ENABLED=true
-export KB_LLM_BASE_URL="https://open.bigmodel.cn/api/paas/v4/"
-export KB_LLM_API_KEY="your-zhipu-api-key"
-export KB_LLM_MODEL="glm-4.7"
-```
-
 工具通过 Z.AI 新版 Python SDK 调用中文智谱开放平台 GLM，依赖固定为 `zai-sdk==0.2.2`，客户端固定使用官方中文写法 `from zai import ZhipuAiClient`，`base_url` 使用 `https://open.bigmodel.cn/api/paas/v4/`。工具不再包含旧 `zhipuai` SDK、国际版 `ZaiClient` 或 OpenAI 调用路径，也不 import 项目 `src` 代码。
-
-## 与线上项目的关系
-
-这个工具只产出符合规范的 `*.md` 文件到 `result/`，后续由线上知识库加载流程处理。
-
-建议线上打包时排除整个 `tools/kb_ingest` 目录。

package/app_config.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import os
 from dataclasses import dataclass
 from functools import lru_cache
 from pathlib import Path
@@ -15,7 +14,7 @@ DEFAULT_CONFIG_PATH = CURRENT_DIR / "config" / "config.yaml"
 
 @dataclass(frozen=True)
 class LlmConfig:
-    """LLM 调用配置，支持配置文件和环境变量覆盖。"""
+    """LLM 调用配置。"""
     enabled: bool = False
     base_url: str = ""
     api_key: str = ""
@@ -35,67 +34,56 @@ class DraftConfig:
 
 @lru_cache(maxsize=1)
 def get_llm_config() -> LlmConfig:
-    """读取并合并 LLM 配置。"""
+    """读取 LLM 配置。"""
     raw = _read_config().get("llm", {})
     if not isinstance(raw, dict):
         raw = {}
 
     return LlmConfig(
-        enabled=_env_bool("KB_LLM_ENABLED", _as_bool(raw.get("enabled"), False)),
-        base_url=os.environ.get("KB_LLM_BASE_URL", str(raw.get("base_url") or "")),
-        api_key=os.environ.get("KB_LLM_API_KEY", str(raw.get("api_key") or "")),
-        model=os.environ.get("KB_LLM_MODEL", str(raw.get("model") or "")),
-        timeout_seconds=_env_int("KB_LLM_TIMEOUT_SECONDS", raw.get("timeout_seconds"), 120),
-        max_tokens=_env_int("KB_LLM_MAX_TOKENS", raw.get("max_tokens"), 4096),
-        temperature=_env_float("KB_LLM_TEMPERATURE", raw.get("temperature"), 0.1),
+        enabled=_as_bool(raw.get("enabled"), False),
+        base_url=str(raw.get("base_url") or ""),
+        api_key=str(raw.get("api_key") or ""),
+        model=str(raw.get("model") or ""),
+        timeout_seconds=_as_int(raw.get("timeout_seconds"), 120),
+        max_tokens=_as_int(raw.get("max_tokens"), 4096),
+        temperature=_as_float(raw.get("temperature"), 0.1),
     )
 
 
 @lru_cache(maxsize=1)
 def get_draft_config() -> DraftConfig:
-    """读取并合并草稿生成配置。"""
+    """读取草稿生成配置。"""
     raw = _read_config().get("draft", {})
     if not isinstance(raw, dict):
         raw = {}
 
     return DraftConfig(
-        max_chars=_env_int("KB_DRAFT_MAX_CHARS", raw.get("max_chars"), 3600),
-        context_chars=_env_int("KB_DRAFT_CONTEXT_CHARS", raw.get("context_chars"), 800),
-        outline_max_sections=_env_int("KB_DRAFT_OUTLINE_MAX_SECTIONS", raw.get("outline_max_sections"), 40),
+        max_chars=_as_int(raw.get("max_chars"), 3600),
+        context_chars=_as_int(raw.get("context_chars"), 800),
+        outline_max_sections=_as_int(raw.get("outline_max_sections"), 40),
     )
 
 
 def _read_config() -> Dict[str, Any]:
     """读取 YAML 配置文件并返回字典。"""
-    path = Path(os.environ.get("KB_INGEST_CONFIG", DEFAULT_CONFIG_PATH))
-    if not path.exists():
+    if not DEFAULT_CONFIG_PATH.exists():
         return {}
-    data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+    data = yaml.safe_load(DEFAULT_CONFIG_PATH.read_text(encoding="utf-8")) or {}
     return data if isinstance(data, dict) else {}
 
 
-def _env_bool(name: str, default: bool) -> bool:
-    """读取布尔环境变量并回退到默认值。"""
-    value = os.environ.get(name)
-    if value is None:
-        return default
-    return _as_bool(value, default)
-
-
-def _env_int(name: str, value: Any, default: int) -> int:
-    """读取整数环境变量并回退到默认值。"""
-    raw = os.environ.get(name, value)
+def _as_int(value: Any, default: int) -> int:
+    """把配置值转换为整数。"""
     try:
-        return int(raw)
+        return int(value)
     except (TypeError, ValueError):
         return default
 
 
-def _env_float(name: str, value: Any, default: float) -> float:
-    """读取浮点环境变量并回退到默认值。"""
-    raw = os.environ.get(name, value)
+def _as_float(value: Any, default: float) -> float:
+    """把配置值转换为浮点数。"""
     try:
-        return float(raw)
+        return float(value)
     except (TypeError, ValueError):
         return default
 

package/config/config.yaml

@@ -1,5 +1,5 @@
 llm:
-  enabled: true
+  enabled: false
   timeout_seconds: 120
   max_tokens: 8192
   temperature: 0.1

package/ingest.py

@@ -10,12 +10,14 @@ from pathlib import Path
 from typing import List
 from uuid import uuid4
 
+import yaml
+
 CURRENT_DIR = Path(__file__).resolve().parent
 if str(CURRENT_DIR) not in sys.path:
     sys.path.insert(0, str(CURRENT_DIR))
 
 from app_config import get_draft_config
-from normalizer import normalize_block
+from normalizer import fallback_failed_block, normalize_block
 from parser import iter_input_files, parse_document
 from schemas import ParsedBlock
 from splitter import split_blocks
@@ -58,6 +60,8 @@ def cmd_draft(args) -> int:
             return 0
         if action == "rebuild":
             _clear_generated_files(output_dir)
+        elif action == "retry":
+            return _retry_failed_files(output_dir, status=args.status)
         elif action == "resume":
             resume_state = _load_progress(progress_path)
             if not resume_state:
@@ -114,39 +118,22 @@ def cmd_draft(args) -> int:
             try:
                 items = normalize_block(block, status=args.status)
             except SystemExit as exc:
-                _save_progress(
-                    progress_path,
-                    input_path=input_path,
-                    output_dir=output_dir,
-                    files=files,
-                    run_timestamp=run_timestamp,
-                    run_trace_id=run_trace_id,
-                    source_order=source_order,
-                    total_items=total_items,
-                    file_index=file_index,
-                    block_index=block_index,
-                    status="failed",
-                    error=f"SystemExit({exc.code})",
-                )
-                print(
-                    "checkpoint saved. "
-                    f"file={path.name} block={block_index + 1}/{len(blocks)} "
-                    f"progress={progress_path}"
-                )
-                raise
-            for item in items:
-                source_order += 1
-                item.source_order = source_order
-                item.source_pages = sorted(set(block.pages))
-                item.source_trace = _source_trace(block)
-                write_item(
-                    item,
-                    output_dir,
-                    source_title=Path(block.source_doc).stem,
-                    timestamp=run_timestamp,
-                    trace_id=run_trace_id,
-                )
-                total_items += 1
+                print(f"WARNING: block failed with SystemExit({exc.code}); writing failed fallback")
+                items = fallback_failed_block(block, status=args.status)
+            except Exception as exc:
+                print(f"WARNING: block failed with {type(exc).__name__}: {exc}; writing failed fallback")
+                items = fallback_failed_block(block, status=args.status)
+            if not items:
+                items = fallback_failed_block(block, status=args.status)
+            source_order, written = _write_items(
+                items,
+                output_dir,
+                block,
+                source_order=source_order,
+                run_timestamp=run_timestamp,
+                run_trace_id=run_trace_id,
+            )
+            total_items += written
             _save_progress(
                 progress_path,
                 input_path=input_path,
@@ -178,20 +165,144 @@ def _list_effective_files(path: Path) -> list[Path]:
     )
 
 
+def _write_items(
+    items,
+    output_dir: Path,
+    block: ParsedBlock,
+    *,
+    source_order: int,
+    run_timestamp: str,
+    run_trace_id: str,
+) -> tuple[int, int]:
+    written = 0
+    for item in items:
+        source_order += 1
+        item.source_order = source_order
+        item.source_pages = sorted(set(block.pages))
+        item.source_trace = _source_trace(block)
+        write_item(
+            item,
+            output_dir,
+            source_title=Path(block.source_doc).stem,
+            timestamp=run_timestamp,
+            trace_id=run_trace_id,
+        )
+        written += 1
+    return source_order, written
+
+
+def _retry_failed_files(output_dir: Path, status: str) -> int:
+    failed_files = [path for path in _list_effective_files(output_dir) if "failed" in path.stem.lower()]
+    if not failed_files:
+        print("done. failed_files=0 retried=0")
+        return 0
+
+    run_timestamp = _make_timestamp()
+    run_trace_id = uuid4().hex[:8]
+    retried = 0
+    succeeded = 0
+    still_failed = 0
+    for path in failed_files:
+        block = _block_from_failed_file(path)
+        if not block:
+            print(f"WARNING: skipped failed file without chunk source: {path}")
+            continue
+        try:
+            items = normalize_block(block, status=status)
+        except SystemExit as exc:
+            print(f"WARNING: retry failed with SystemExit({exc.code}); keeping failed fallback: {path}")
+            items = fallback_failed_block(block, status=status)
+        except Exception as exc:
+            print(f"WARNING: retry failed with {type(exc).__name__}: {exc}; keeping failed fallback: {path}")
+            items = fallback_failed_block(block, status=status)
+        if not items:
+            items = fallback_failed_block(block, status=status)
+
+        source_order = max(int(block.order or 0) - 1, 0)
+        _, written = _write_items(
+            items,
+            output_dir,
+            block,
+            source_order=source_order,
+            run_timestamp=run_timestamp,
+            run_trace_id=run_trace_id,
+        )
+        if written:
+            path.unlink()
+        retried += 1
+        if any(item.review_status == "failed" for item in items):
+            still_failed += 1
+        else:
+            succeeded += 1
+    print(f"done. failed_files={len(failed_files)} retried={retried} succeeded={succeeded} still_failed={still_failed}")
+    return 0
+
+
+def _block_from_failed_file(path: Path) -> ParsedBlock | None:
+    text = path.read_text(encoding="utf-8")
+    metadata = _front_matter(text)
+    source = _failed_chunk_source(text)
+    if not source:
+        return None
+    return ParsedBlock(
+        source_doc=str(metadata.get("source_doc") or path.stem),
+        source_section=str(metadata.get("source_section") or ""),
+        content=source,
+        pages=[int(page) for page in metadata.get("source_pages") or [] if str(page).isdigit()],
+        order=int(metadata.get("source_order") or 0),
+        category=str(metadata.get("category") or ""),
+        category_keywords=[str(item) for item in metadata.get("category_keywords") or []],
+        source_doc_description=str(metadata.get("source_doc_description") or ""),
+        subcategory=str(metadata.get("subcategory") or ""),
+        category_path=[str(item) for item in metadata.get("category_path") or []],
+        related_categories=[str(item) for item in metadata.get("related_categories") or []],
+        relation_notes=[str(item) for item in metadata.get("relation_notes") or []],
+        related_items=metadata.get("related_items") or [],
+    )
+
+
+def _front_matter(text: str) -> dict:
+    if not text.startswith("---\n"):
+        return {}
+    end = text.find("\n---\n", 4)
+    if end < 0:
+        return {}
+    data = yaml.safe_load(text[4:end]) or {}
+    return data if isinstance(data, dict) else {}
+
+
+def _failed_chunk_source(text: str) -> str:
+    marker = "## failed_chunk_source"
+    start = text.find(marker)
+    if start < 0:
+        return ""
+    source = text[start + len(marker):].strip()
+    if source.startswith("```"):
+        first_line = source.find("\n")
+        if first_line >= 0:
+            source = source[first_line + 1:]
+        if source.endswith("```"):
+            source = source[:-3]
+    return source.strip()
+
+
 def _choose_existing_result_action(output_dir: Path, existing: list[Path]) -> str:
     """询问用户如何处理已有生成结果。"""
     print(f"found {len(existing)} existing file(s) in {output_dir}.")
     print("Choose how to continue:")
     print("1. delete and rebuild")
     print("2. resume from checkpoint")
-    print("3. exit")
-    answer = input("Select [1/2/3]: ").strip().lower().translate(
-        str.maketrans({"１": "1", "２": "2", "３": "3"})
+    print("3. retry failed files")
+    print("4. exit")
+    answer = input("Select [1/2/3/4]: ").strip().lower().translate(
+        str.maketrans({"１": "1", "２": "2", "３": "3", "４": "4"})
     )
     if answer.startswith("1") or answer in {"d", "delete", "rebuild", "r"}:
         return "rebuild"
     if answer.startswith("2") or answer in {"resume", "continue", "c"}:
         return "resume"
+    if answer.startswith("3") or answer in {"retry", "failed", "f"}:
+        return "retry"
     return "exit"
 
 

package/input/function/.gitkeep

@@ -0,0 +1 @@
+

package/input/function/tools.yaml

@@ -0,0 +1,27 @@
+tools:
+  - name: "query_member_change_announcements"
+    display_name: "查询成员机构变更通知"
+    description: "查询成员机构近期变更、变更报备、计划变更、执行评价以及影响范围。"
+    trigger: "当用户询问成员机构变更通知、变更报备、计划变更或执行评价时优先调用。"
+    input_schema:
+      type: "object"
+      properties:
+        orgCode:
+          type: "string"
+          description: "成员机构金融编码，通常由 resolve_member_org 工具获得。"
+      required: ["orgCode"]
+    output_schema:
+      type: "array"
+      description: "匹配到的成员机构变更通知列表。"
+      items:
+        type: "object"
+        properties:
+          member_code:
+            type: "string"
+            description: "成员机构编码。"
+          member_name:
+            type: "string"
+            description: "成员机构名称。"
+          change_type:
+            type: "string"
+            description: "变更类型。"

package/normalizer.py

@@ -15,8 +15,8 @@ from app_config import get_llm_config
 from schemas import DOC_TYPES, KnowledgeItem, ParsedBlock
 
 
-DEFAULT_DOMAIN = "网联清算业务"
-DEFAULT_OWNER = "网联清算业务知识库"
+DEFAULT_DOMAIN = "通用业务"
+DEFAULT_OWNER = "通用知识库"
 CURRENT_DIR = Path(__file__).resolve().parent
 KB_SPEC_PATH = CURRENT_DIR / "prompts" / "知识库建立规范.md"
 TOOLS_PATH = CURRENT_DIR / "input" / "function" / "tools.yaml"
@@ -37,15 +37,23 @@ def normalize_block(block: ParsedBlock, status: str = "draft") -> List[Knowledge
         items = _normalize_with_llm(block, status=status)
         if items:
             return [_postprocess_item(item, block) for item in items]
-        _abort_llm("model returned no valid knowledge items", block)
+        print("WARNING: model returned no valid knowledge items; using heuristic fallback")
+        return fallback_failed_block(block, status=status)
     return [_postprocess_item(_normalize_heuristically(block, status=status), block)]
 
 
+def fallback_failed_block(block: ParsedBlock, status: str = "draft") -> List[KnowledgeItem]:
+    """用离线规则生成 failed 兜底条目。"""
+    item = _normalize_heuristically(block, status=status)
+    return [_mark_llm_failed_item(_postprocess_item(item, block), block)]
+
+
 def _normalize_with_llm(block: ParsedBlock, status: str) -> List[KnowledgeItem]:
     """调用 LLM 生成条目并处理重试。"""
     config = get_llm_config()
     if not (config.base_url and config.api_key and config.model):
-        _abort_llm("missing base_url, api_key, or model", block)
+        print("WARNING: missing base_url, api_key, or model; using heuristic fallback")
+        return []
 
     base_prompt = _build_prompt(block, status)
     started_at = time.monotonic()
@@ -61,22 +69,15 @@ def _normalize_with_llm(block: ParsedBlock, status: str) -> List[KnowledgeItem]:
         return []
     client = client_cls(api_key=config.api_key, base_url=config.base_url)
 
-    compact_retry = False
     coverage_retry_count = 0
-    coverage_retry_feedback = ""
-    json_retry_feedback = ""
+    messages = _messages(base_prompt)
     for attempt in range(1, LLM_MAX_RETRIES + 1):
-        prompt = _compact_retry_prompt(base_prompt) if compact_retry else base_prompt
-        if json_retry_feedback:
-            prompt = f"{prompt}\n\n{json_retry_feedback}"
-        if coverage_retry_feedback:
-            prompt = f"{prompt}\n\n{coverage_retry_feedback}"
         try:
             print(
                 "llm request: "
                 f"provider=zhipu base_url={config.base_url} attempt={attempt}/{LLM_MAX_RETRIES}"
             )
-            response = _create_zhipu_completion(client, config, prompt)
+            response = _create_zhipu_completion(client, config, messages)
         except Exception as exc:
             elapsed = time.monotonic() - started_at
             print(
@@ -85,7 +86,8 @@ def _normalize_with_llm(block: ParsedBlock, status: str) -> List[KnowledgeItem]:
                 f"after {elapsed:.1f}s detail={exc}"
             )
             if attempt >= LLM_MAX_RETRIES:
-                _abort_llm(f"request failed after {LLM_MAX_RETRIES} attempts: {type(exc).__name__}", block)
+                print(f"WARNING: request failed after {LLM_MAX_RETRIES} attempts: {type(exc).__name__}; using heuristic fallback")
+                return []
             time.sleep(min(2 ** (attempt - 1), 30))
             continue
 
@@ -107,14 +109,22 @@ def _normalize_with_llm(block: ParsedBlock, status: str) -> List[KnowledgeItem]:
                 f"response={_response_debug(response)}"
             )
             if attempt >= LLM_MAX_RETRIES:
-                _abort_llm("empty response content after 10 attempts", block)
+                print(f"WARNING: empty response content after {LLM_MAX_RETRIES} attempts; using heuristic fallback")
+                return []
+            _append_retry_messages(
+                messages,
+                content,
+                "重试补充要求：上一轮响应为空。请只返回一个 JSON object，根节点必须严格为 {\"items\": [...]}。",
+            )
             time.sleep(min(2 ** (attempt - 1), 30))
             continue
 
         parse_result = _extract_json_with_diagnostics(content)
         parsed = parse_result.value
         if not parsed:
-            compact_retry = compact_retry or _looks_truncated(content, finish_reason)
+            retry_prompt = _json_repair_retry_prompt(parse_result.error, content)
+            if _looks_truncated(content, finish_reason):
+                retry_prompt += "\n上一轮响应疑似被截断，本轮请压缩表述但保持 {\"items\": [...]} 根结构。"
             print(
                 "llm parse failed: response is not valid JSON "
                 f"finish_reason={finish_reason or 'unknown'} "
@@ -123,15 +133,17 @@ def _normalize_with_llm(block: ParsedBlock, status: str) -> List[KnowledgeItem]:
                 f"preview={_preview(content)}"
             )
             if attempt >= LLM_MAX_RETRIES:
-                _abort_llm("response is not valid JSON after 10 attempts", block)
-            json_retry_feedback = _json_repair_retry_prompt(parse_result.error, content)
+                print(f"WARNING: response is not valid JSON after {LLM_MAX_RETRIES} attempts; using heuristic fallback")
+                return []
+            _append_retry_messages(messages, content, retry_prompt)
             time.sleep(min(2 ** (attempt - 1), 30))
             continue
-        json_retry_feedback = ""
 
         raw_items = _coerce_raw_items(parsed)
         if not isinstance(raw_items, list):
-            compact_retry = compact_retry or _looks_truncated(content, finish_reason)
+            retry_prompt = _json_shape_retry_prompt(parsed)
+            if _looks_truncated(content, finish_reason):
+                retry_prompt += "\n上一轮响应疑似被截断，本轮请压缩表述但保持 {\"items\": [...]} 根结构。"
             print(
                 "llm parse failed: JSON does not contain an items list "
                 f"finish_reason={finish_reason or 'unknown'} "
@@ -139,11 +151,11 @@ def _normalize_with_llm(block: ParsedBlock, status: str) -> List[KnowledgeItem]:
                 f"top_level={_json_shape(parsed)} preview={_preview(content)}"
             )
             if attempt >= LLM_MAX_RETRIES:
-                _abort_llm("JSON does not contain an items list after 10 attempts", block)
-            json_retry_feedback = _json_shape_retry_prompt(parsed)
+                print(f"WARNING: JSON does not contain an items list after {LLM_MAX_RETRIES} attempts; using heuristic fallback")
+                return []
+            _append_retry_messages(messages, content, retry_prompt)
             time.sleep(min(2 ** (attempt - 1), 30))
             continue
-        json_retry_feedback = ""
 
         items: List[KnowledgeItem] = []
         for idx, raw in enumerate(raw_items, start=1):
@@ -154,37 +166,37 @@ def _normalize_with_llm(block: ParsedBlock, status: str) -> List[KnowledgeItem]:
         if items:
             coverage_issues = _source_fact_coverage_issues(block, items)
             if coverage_issues:
-                high_relevance_issues = _high_relevance_coverage_issues(
-                    client, config, block, items, coverage_issues
-                )
-                if not high_relevance_issues:
-                    print(
-                        "llm coverage warning ignored: "
-                        "no highly relevant missing facts after relevance review"
-                    )
-                    return items
                 print(
                     "llm coverage failed: "
                     f"missing_facts={len(coverage_issues)} "
-                    f"high_relevance_missing_facts={len(high_relevance_issues)} "
-                    f"preview={_preview('；'.join(high_relevance_issues[:3]))}"
+                    f"preview={_preview('；'.join(coverage_issues[:3]))}"
                 )
                 if coverage_retry_count >= COVERAGE_MAX_RETRIES or attempt >= LLM_MAX_RETRIES:
                     print(
                         "WARNING: source fact coverage failed after "
                         f"{coverage_retry_count} coverage retries; releasing draft for manual review"
                     )
-                    return _items_with_coverage_warning(items, block, high_relevance_issues)
+                    return _items_with_coverage_warning(items, block, coverage_issues)
                 coverage_retry_count += 1
-                coverage_retry_feedback = _coverage_retry_prompt(block, high_relevance_issues, items)
+                _append_retry_messages(
+                    messages,
+                    content,
+                    _coverage_retry_prompt(block, coverage_issues, items),
+                )
                 time.sleep(min(2 ** (attempt - 1), 30))
                 continue
             return items
         if attempt >= LLM_MAX_RETRIES:
-            _abort_llm("items list contained no valid objects after 10 attempts", block)
+            print(f"WARNING: items list contained no valid objects after {LLM_MAX_RETRIES} attempts; using heuristic fallback")
+            return []
+        _append_retry_messages(
+            messages,
+            content,
+            "重试补充要求：上一轮 items 数组没有可用对象。请返回 {\"items\": [...]}，items 中每个元素都必须是知识库条目对象。",
+        )
         time.sleep(min(2 ** (attempt - 1), 30))
 
-    _abort_llm("model call failed", block)
+    return []
 
 
 def _system_message() -> str:
@@ -204,176 +216,17 @@ def _messages(prompt: str) -> List[Dict[str, str]]:
     ]
 
 
-def _compact_retry_prompt(base_prompt: str) -> str:
-    """构造输出截断后的紧凑重试提示。"""
-    return (
-        base_prompt
-        + "\n\n重试补充要求：上一次输出疑似过长或结构不完整。"
-        "本次必须只生成 1 个 item，保留规范要求的正文 5 个章节，但不能丢失原文事实句。"
-        "不要省略 JSON 外层 items，不要输出多个条目，不要输出解释文字。"
-    )
 
 
-def _high_relevance_coverage_issues(
-    client,
-    config,
-    block: ParsedBlock,
-    items: List[KnowledgeItem],
-    missing_facts: List[str],
-) -> List[str]:
-    """让 LLM 判断缺失事实是否与当前条目高度相关。"""
-    if not missing_facts:
-        return []
-    prompt = _coverage_relevance_prompt(block, items, missing_facts)
-    try:
-        response = _create_zhipu_completion(client, config, prompt)
-        content = _extract_response_content(response)
-        parsed = _extract_json_with_diagnostics(content).value
-        high_relevance = _high_relevance_facts_from_analysis(parsed, missing_facts)
-        if high_relevance is not None:
-            print(
-                "llm coverage relevance: "
-                f"missing_facts={len(missing_facts)} high_relevance={len(high_relevance)}"
-            )
-            return high_relevance
-    except Exception as exc:
-        print(f"llm coverage relevance failed: {type(exc).__name__} detail={exc}")
 
-    fallback = _fallback_high_relevance_coverage_issues(block, items, missing_facts)
-    print(
-        "llm coverage relevance fallback: "
-        f"missing_facts={len(missing_facts)} high_relevance={len(fallback)}"
-    )
-    return fallback
 
 
-def _coverage_relevance_prompt(
-    block: ParsedBlock,
-    items: List[KnowledgeItem],
-    missing_facts: List[str],
-) -> str:
-    """构造缺失事实相关性判定提示。"""
-    fact_lines = "\n".join(f"- {fact}" for fact in missing_facts[:20])
-    current_items = "\n\n".join(
-        f"标题：{item.title}\n核心正文：{_core_sections_for_coverage(item.body)}"
-        for item in items
-    )
-    return f"""
-请判断以下“覆盖校验缺失事实”是否与当前知识条目的主题极高相关。
-
-判定规则：
-1. 只有缺失事实是回答当前条目标题或核心正文所必须保留的定义、规则、阈值、条件、主体、简称、例外或限制时，才标记为“极高”。
-2. 来源文件标题、章节标题、目录项、上级主题名称、页眉页脚、纯标签、仅用于定位的小标题，通常不是“极高”，除非它本身就是当前条目要解释的完整定义或规则。
-3. 辅助上下文只用于理解位置和主题，不要把辅助上下文中独有的信息作为缺失事实依据。
-4. 只能返回 JSON object，不要 Markdown 或解释文字。
-
-返回格式：
-{{
-  "facts": [
-    {{"fact": "必须原样复制待判断事实", "relevance": "极高|一般|低", "reason": "一句话原因"}}
-  ]
-}}
-
-来源文档：{block.source_doc}
-来源章节：{block.source_section or "全文"}
-
-当前来源原文片段：
-{_preview(block.content)[:4000] or "无"}
-
-辅助上下文：
-{_preview(block.context)[:2000] or "无"}
-
-当前已生成条目：
-{_preview(current_items)[:4000] or "无"}
-
-待判断事实：
-{fact_lines}
-""".strip()
 
 
-def _high_relevance_facts_from_analysis(parsed, missing_facts: List[str]):
-    """从相关性判定 JSON 中提取极高相关事实。"""
-    if not isinstance(parsed, dict):
-        return None
-    raw_facts = parsed.get("facts")
-    if raw_facts is None and isinstance(parsed.get("results"), list):
-        raw_facts = parsed.get("results")
-    if raw_facts is None and isinstance(parsed.get("items"), list):
-        raw_facts = parsed.get("items")
-    if not isinstance(raw_facts, list):
-        return None
-
-    missing_by_norm = {_coverage_text(fact): fact for fact in missing_facts}
-    selected: List[str] = []
-    for raw in raw_facts:
-        if not isinstance(raw, dict):
-            continue
-        relevance = str(raw.get("relevance") or raw.get("关联度") or "").strip().lower()
-        if not ("极高" in relevance or "high" in relevance):
-            continue
-        fact = str(raw.get("fact") or raw.get("事实") or raw.get("text") or "").strip()
-        matched = _match_missing_fact(fact, missing_by_norm)
-        if matched and matched not in selected:
-            selected.append(matched)
-    return selected
 
 
-def _match_missing_fact(fact: str, missing_by_norm: Dict[str, str]) -> str:
-    """把模型返回事实匹配回原始缺失事实。"""
-    fact_norm = _coverage_text(fact)
-    if not fact_norm:
-        return ""
-    if fact_norm in missing_by_norm:
-        return missing_by_norm[fact_norm]
-    for missing_norm, missing in missing_by_norm.items():
-        if fact_norm in missing_norm or missing_norm in fact_norm:
-            return missing
-    return ""
 
 
-def _fallback_high_relevance_coverage_issues(
-    block: ParsedBlock,
-    items: List[KnowledgeItem],
-    missing_facts: List[str],
-) -> List[str]:
-    """相关性判定失败时的保守兜底，过滤明显结构性标题。"""
-    return [
-        fact for fact in missing_facts
-        if not _looks_like_structural_missing_fact(block, items, fact)
-    ]
-
-
-def _looks_like_structural_missing_fact(
-    block: ParsedBlock,
-    items: List[KnowledgeItem],
-    fact: str,
-) -> bool:
-    """判断缺失事实是否只是标题、章节或定位信息。"""
-    fact_norm = _coverage_text(fact)
-    if not fact_norm:
-        return True
-    candidates = [
-        block.source_doc,
-        Path(block.source_doc).stem,
-        block.source_section,
-        block.category,
-        block.subcategory,
-        block.source_doc_description,
-        block.subcategory_description,
-        *block.category_path,
-        *block.related_categories,
-        *(item.title for item in items),
-    ]
-    candidate_norms = {_coverage_text(value) for value in candidates if value}
-    if fact_norm in candidate_norms:
-        return True
-    if len(fact_norm) <= 30 and not re.search(
-        r"是|为|指|称|简称|英文|应|需|必须|不得|禁止|超过|低于|大于|小于|不少于|不超过|\d",
-        fact,
-    ):
-        return True
-    return False
-
 
 def _coverage_retry_prompt(
     block: ParsedBlock,
@@ -427,17 +280,23 @@ def _json_shape_retry_prompt(parsed) -> str:
     )
 
 
-def _create_zhipu_completion(client, config, prompt: str):
+def _append_retry_messages(messages: List[Dict[str, str]], previous_content: str, feedback: str) -> None:
+    """把坏返回和纠偏要求放进下一轮会话。"""
+    messages.append({"role": "assistant", "content": previous_content or ""})
+    messages.append({"role": "user", "content": feedback})
+
+
+def _create_zhipu_completion(client, config, messages: List[Dict[str, str]]):
     """发起一次非流式模型补全请求。"""
     return client.chat.completions.create(
         model=config.model,
-        messages=_messages(prompt),
+        messages=messages,
         stream=False,
         max_tokens=config.max_tokens,
         temperature=config.temperature,
         do_sample=False,
         response_format={"type": "json_object"},
-        thinking={"type": "disabled", "clear_thinking": True},
+        thinking={"type": "disabled", "clear_thinking": False},
     )
 
 
@@ -574,69 +433,35 @@ def _json_shape(value) -> str:
     return type(value).__name__
 
 
-def _coerce_raw_items(parsed):
-    """把兼容 JSON 结构转换为 items 列表。"""
-    if isinstance(parsed, dict):
-        items = parsed.get("items")
-        if isinstance(items, list):
-            return items
+def _mark_llm_failed_item(item: KnowledgeItem, block: ParsedBlock) -> KnowledgeItem:
+    """标记 LLM 失败后的离线兜底条目。"""
+    item.review_status = "failed"
+    item.body = _append_failed_chunk_source(item.body, block.content)
+    return item
+
+
+def _append_failed_chunk_source(body: str, source: str) -> str:
+    """在 failed 文件末尾保存原始 chunk。"""
+    warning = (
+        "## LLM 生成失败警告\n\n"
+        "WARNING: LLM 多次重试后仍未返回合格 JSON，本文件由离线规则兜底生成，需人工核对。\n\n"
+        "## failed_chunk_source\n\n"
+        "```text\n"
+        f"{source.strip()}\n"
+        "```"
+    )
+    return f"{body.strip()}\n\n{warning}".strip()
 
-        for key in (
-            "knowledge_items",
-            "records",
-            "data",
-            "payload",
-            "output",
-            "response",
-            "answer",
-            "content",
-            "message",
-            "result",
-            "results",
-        ):
-            value = parsed.get(key)
-            if isinstance(value, list):
-                print(f"llm parse notice: using non-standard list field '{key}' as items")
-                return value
-            if isinstance(value, dict):
-                nested = _coerce_raw_items(value)
-                if isinstance(nested, list):
-                    print(f"llm parse notice: using nested field '{key}' as items")
-                    return nested
-            if isinstance(value, str) and value.strip():
-                nested = _extract_json_with_diagnostics(value)
-                if nested.value is not None:
-                    nested_items = _coerce_raw_items(nested.value)
-                    if isinstance(nested_items, list):
-                        print(f"llm parse notice: parsed JSON string field '{key}' as items")
-                        return nested_items
-
-        if _looks_like_single_item(parsed):
-            print("llm parse notice: wrapping single item object as items[0]")
-            return [parsed]
-
-    if isinstance(parsed, list):
-        print("llm parse notice: wrapping root array as items")
-        return parsed
 
+def _coerce_raw_items(parsed):
+    """只接受标准 {"items": [...]} 输出。"""
+    if isinstance(parsed, dict) and isinstance(parsed.get("items"), list):
+        return parsed["items"]
     return None
 
 
-def _looks_like_single_item(value: Dict) -> bool:
-    """判断 JSON 对象是否像单个条目。"""
-    required_signal = {"title", "body"}
-    item_fields = {
-        "title",
-        "doc_type",
-        "business_modules",
-        "source_version",
-        "risk_level",
-        "applicable_roles",
-        "tags",
-        "body",
-        "split_reason",
-    }
-    return required_signal.issubset(value.keys()) and len(item_fields.intersection(value.keys())) >= 4
+
+
 
 
 def _looks_truncated(content: str, finish_reason: str) -> bool:
@@ -649,15 +474,6 @@ def _looks_truncated(content: str, finish_reason: str) -> bool:
     return stripped.count("{") > stripped.count("}") or stripped.count("[") > stripped.count("]")
 
 
-def _abort_llm(message: str, block: ParsedBlock) -> None:
-    """输出失败告警并终止 LLM 生成。"""
-    print(
-        "ALERT: llm draft failed; aborting. "
-        f"reason={message} doc={block.source_doc} section={block.source_section}"
-    )
-    raise SystemExit(1)
-
-
 def _build_prompt(block: ParsedBlock, status: str) -> str:
     """构造片段生成知识条目的完整提示。"""
     spec = _read_kb_spec()
@@ -757,38 +573,23 @@ def _read_tools() -> List[Dict[str, Any]]:
 
 def _extract_json_with_diagnostics(text: str) -> JsonParseResult:
     """提取 JSON 并保留失败诊断。"""
-    text = text.strip()
-    text = _strip_code_fence(text)
+    text = _strip_code_fence(text.strip())
     errors: List[str] = []
     try:
         return JsonParseResult(value=json.loads(text))
     except json.JSONDecodeError as exc:
         errors.append(_json_error_message(exc, text))
 
-    repaired = _repair_json_text(text)
-    if repaired != text:
-        try:
-            print("llm parse notice: repaired unescaped quotes in JSON string fields")
-            return JsonParseResult(value=json.loads(repaired))
-        except json.JSONDecodeError as exc:
-            errors.append(_json_error_message(exc, repaired))
-
     for candidate in _json_candidates(text):
         try:
             return JsonParseResult(value=json.loads(candidate))
         except json.JSONDecodeError as exc:
             errors.append(_json_error_message(exc, candidate))
-            repaired = _repair_json_text(candidate)
-            if repaired != candidate:
-                try:
-                    print("llm parse notice: repaired unescaped quotes in JSON candidate")
-                    return JsonParseResult(value=json.loads(repaired))
-                except json.JSONDecodeError as repair_exc:
-                    errors.append(_json_error_message(repair_exc, repaired))
-                    continue
     return JsonParseResult(error=errors[-1] if errors else "no JSON object or array found")
 
 
+
+
 def _json_error_message(exc: json.JSONDecodeError, text: str) -> str:
     """格式化 JSON 解析错误位置。"""
     line = text.splitlines()[exc.lineno - 1] if 0 < exc.lineno <= len(text.splitlines()) else ""
@@ -823,130 +624,18 @@ def _json_candidates(text: str) -> List[str]:
     return candidates
 
 
-def _repair_json_text(text: str) -> str:
-    """修复常见的 JSON 字符串引号问题。"""
-    for field in (
-        "title",
-        "category",
-        "category_description",
-        "source_version",
-        "risk_level",
-        "split_reason",
-    ):
-        text = _repair_unescaped_quotes_in_json_field(text, field)
-    return text
 
 
-def _repair_unescaped_quotes_in_json_field(text: str, field: str) -> str:
-    """修复指定 JSON 字段中的未转义引号。"""
-    pattern = re.compile(rf'("{re.escape(field)}"\s*:\s*)"')
-    output = []
-    cursor = 0
-    while True:
-        match = pattern.search(text, cursor)
-        if not match:
-            output.append(text[cursor:])
-            break
 
-        value_start = match.end()
-        closing = _find_json_field_string_end(text, value_start)
-        if closing == -1:
-            output.append(text[cursor:])
-            break
 
-        raw_value = text[value_start:closing]
-        repaired_value = _escape_unescaped_quotes(raw_value)
-        output.append(text[cursor:value_start])
-        output.append(repaired_value)
-        output.append('"')
-        cursor = closing + 1
-
-    return "".join(output)
-
-
-def _find_json_field_string_end(text: str, start: int) -> int:
-    """定位 JSON 字符串字段的结束引号。"""
-    idx = start
-    escaped = False
-    while idx < len(text):
-        char = text[idx]
-        if escaped:
-            escaped = False
-        elif char == "\\":
-            escaped = True
-        elif char == '"' and _looks_like_json_value_end(text, idx):
-            return idx
-        idx += 1
-    return -1
-
-
-def _looks_like_json_value_end(text: str, quote_idx: int) -> bool:
-    """判断引号是否像 JSON 值结尾。"""
-    next_idx = _next_nonspace(text, quote_idx + 1)
-    if next_idx >= len(text):
-        return True
-    if text[next_idx] in "}]":
-        return True
-    if text[next_idx] != ",":
-        return False
 
-    following = _next_nonspace(text, next_idx + 1)
-    if following >= len(text):
-        return True
-    if text[following] in "}]":
-        return True
-    if text[following] != '"':
-        return False
 
-    key_end = _find_plain_json_string_end(text, following + 1)
-    if key_end == -1:
-        return False
-    after_key = _next_nonspace(text, key_end + 1)
-    return after_key < len(text) and text[after_key] == ":"
-
-
-def _next_nonspace(text: str, start: int) -> int:
-    """查找下一个非空白字符位置。"""
-    idx = start
-    while idx < len(text) and text[idx].isspace():
-        idx += 1
-    return idx
-
-
-def _find_plain_json_string_end(text: str, start: int) -> int:
-    """定位普通 JSON 字符串结束位置。"""
-    idx = start
-    escaped = False
-    while idx < len(text):
-        char = text[idx]
-        if escaped:
-            escaped = False
-        elif char == "\\":
-            escaped = True
-        elif char == '"':
-            return idx
-        idx += 1
-    return -1
-
-
-def _escape_unescaped_quotes(value: str) -> str:
-    """转义字符串中的裸双引号。"""
-    output = []
-    escaped = False
-    for char in value:
-        if escaped:
-            output.append(char)
-            escaped = False
-            continue
-        if char == "\\":
-            output.append(char)
-            escaped = True
-            continue
-        if char == '"':
-            output.append('\\"')
-            continue
-        output.append(char)
-    return "".join(output)
+
+
+
+
+
+
 
 
 def _preview(text: str) -> str:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "union_kb_ingest",
-  "version": "1.0.10",
+  "version": "1.0.11",
   "description": "Offline knowledge-base ingest helper for PDF, Word, Markdown and TXT documents.",
   "bin": {
     "union_kb_ingest": "bin/union_kb_ingest"
@@ -9,12 +9,13 @@
     "README.md",
     "requirements.txt",
     "bin/union_kb_ingest",
-    "ArkKickidcService.java",
     "*.py",
     "config/config.yaml",
     "prompts/",
     "input/.gitkeep",
     "input/pdf/.gitkeep",
+    "input/function/.gitkeep",
+    "input/function/tools.yaml",
     "input/word/.gitkeep",
     "parsed/.gitkeep",
     "result/.gitkeep"

package/requirements.txt

@@ -1,9 +1,6 @@
-# Offline-only optional dependencies. Do not add these to the app runtime unless needed.
 pyyaml>=6.0.1
 zai-sdk==0.2.2
-sniffio>=1.3.0
 
-# Docling slim plus only file-format backends used by this offline tool.
-# Do not install `docling` or `docling-slim[standard]` here: those pull OCR,
-# layout/table ML models, torch/onnxruntime, and may try to download artifacts.
+# Docling file-format backends only. Avoid docling-slim[standard]: it pulls OCR,
+# layout/table ML models, torch/onnxruntime, and downloaded artifacts.
 docling-slim[format-pdf-docling,format-docx,format-markdown]>=2.70.0; python_version >= "3.10"

package/writer.py

@@ -22,6 +22,7 @@ def write_item(
         source_title,
         timestamp,
         trace_id,
+        "failed" if item.review_status == "failed" else None,
         _order_prefix(item.source_order),
         item.kb_id,
     ]