union_kb_ingest 1.0.9 → 1.0.11

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Simon
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -12,23 +12,11 @@
 
 启用大模型时，工具只会读取 `prompts/知识库建立规范.md` 作为格式和质量约束，并由代码按当前片段、辅助上下文和输出 JSON 结构组装生成提示词。模型需依据原文语义判断业务场景、模块、角色、标签和风险等级；代码中的启发式生成只作为未启用大模型时的兜底，不使用预设业务关键词去指导大模型输出。
 
-## 安装可选依赖
-
-不建议把这些依赖加入项目根 `requirements.txt`。离线机器单独安装即可：
+## 安装依赖
 
 ```bash
 python -m pip install -r requirements.txt
 ```
-
-解析层不使用 OCR，不加载本地视觉/版面模型，也不访问远程模型服务：
-
-- PDF：使用 `docling-parse` 抽取 PDF 内嵌文本和文本行顺序；扫描件或图片型 PDF 不会识别。
-- DOCX：使用 Docling 的 Word 后端转为 Markdown。
-- 旧版 `.doc`：通过 LibreOffice `soffice` 转为 `.docx` 后再解析；不使用 OCR。
-- Markdown / TXT：作为已文本化材料直接读取。
-
-不要安装 `docling` 或 `docling-slim[standard]`，它们会引入 OCR、版面/表格模型、Torch/ONNXRuntime 等重依赖，并可能在运行时下载模型。内网机器建议为离线工具单独准备 Python 3.10+ 环境。
-
 ## 基本用法
 
 把文件放入：
@@ -37,13 +25,15 @@ python -m pip install -r requirements.txt
 input/
 ```
 
+工具调用示例放在 `input/function/tools.yaml` 中，可按业务需要替换。
+
 生成知识库文件：
 
 ```bash
 python ingest.py draft
 ```
 
-如果 `result/` 中已有生成文件，命令会先询问是否覆盖。选择 `y` 后会清空 `result/` 中已有生成文件，再重新生成；选择其他内容会直接退出，避免多次生成结果相互影响。
+如果 `result/` 中已有生成文件，命令会提示选择删除重建、从断点继续或退出。断点状态保存在 `result/.draft_progress.json`，大模型多次重试失败退出时会记录当前文件和片段位置，下次可选择从断点继续。
 
 只解析为中间 Markdown：
 
@@ -67,7 +57,7 @@ python ingest.py validate
 
 ## 大模型配置
 
-默认不强制调用大模型，会使用启发式模板生成知识库文件。
+默认不强制调用大模型，但是强烈建议启用大模型分析，会使用启发式模板生成知识库文件。
 
 如果要启用大模型整理，修改 `config/config.yaml`：
 
@@ -75,7 +65,7 @@ python ingest.py validate
 llm:
   enabled: true
   base_url: "https://open.bigmodel.cn/api/paas/v4/"
-  api_key: "your-zhipu-api-key"
+  api_key: ""
   model: "glm-4.7"
   timeout_seconds: 120
   max_tokens: 8192
@@ -87,19 +77,4 @@ draft:
   outline_max_sections: 40
 ```
 
-也可以继续使用环境变量覆盖配置文件：
-
-```bash
-export KB_LLM_ENABLED=true
-export KB_LLM_BASE_URL="https://open.bigmodel.cn/api/paas/v4/"
-export KB_LLM_API_KEY="your-zhipu-api-key"
-export KB_LLM_MODEL="glm-4.7"
-```
-
 工具通过 Z.AI 新版 Python SDK 调用中文智谱开放平台 GLM，依赖固定为 `zai-sdk==0.2.2`，客户端固定使用官方中文写法 `from zai import ZhipuAiClient`，`base_url` 使用 `https://open.bigmodel.cn/api/paas/v4/`。工具不再包含旧 `zhipuai` SDK、国际版 `ZaiClient` 或 OpenAI 调用路径，也不 import 项目 `src` 代码。
-
-## 与线上项目的关系
-
-这个工具只产出符合规范的 `*.md` 文件到 `result/`，后续由线上知识库加载流程处理。
-
-建议线上打包时排除整个 `tools/kb_ingest` 目录。

package/app_config.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import os
 from dataclasses import dataclass
 from functools import lru_cache
 from pathlib import Path
@@ -15,7 +14,7 @@ DEFAULT_CONFIG_PATH = CURRENT_DIR / "config" / "config.yaml"
 
 @dataclass(frozen=True)
 class LlmConfig:
-    """LLM 调用配置，支持配置文件和环境变量覆盖。"""
+    """LLM 调用配置。"""
     enabled: bool = False
     base_url: str = ""
     api_key: str = ""
@@ -35,67 +34,56 @@ class DraftConfig:
 
 @lru_cache(maxsize=1)
 def get_llm_config() -> LlmConfig:
-    """读取并合并 LLM 配置。"""
+    """读取 LLM 配置。"""
     raw = _read_config().get("llm", {})
     if not isinstance(raw, dict):
         raw = {}
 
     return LlmConfig(
-        enabled=_env_bool("KB_LLM_ENABLED", _as_bool(raw.get("enabled"), False)),
-        base_url=os.environ.get("KB_LLM_BASE_URL", str(raw.get("base_url") or "")),
-        api_key=os.environ.get("KB_LLM_API_KEY", str(raw.get("api_key") or "")),
-        model=os.environ.get("KB_LLM_MODEL", str(raw.get("model") or "")),
-        timeout_seconds=_env_int("KB_LLM_TIMEOUT_SECONDS", raw.get("timeout_seconds"), 120),
-        max_tokens=_env_int("KB_LLM_MAX_TOKENS", raw.get("max_tokens"), 4096),
-        temperature=_env_float("KB_LLM_TEMPERATURE", raw.get("temperature"), 0.1),
+        enabled=_as_bool(raw.get("enabled"), False),
+        base_url=str(raw.get("base_url") or ""),
+        api_key=str(raw.get("api_key") or ""),
+        model=str(raw.get("model") or ""),
+        timeout_seconds=_as_int(raw.get("timeout_seconds"), 120),
+        max_tokens=_as_int(raw.get("max_tokens"), 4096),
+        temperature=_as_float(raw.get("temperature"), 0.1),
     )
 
 
 @lru_cache(maxsize=1)
 def get_draft_config() -> DraftConfig:
-    """读取并合并草稿生成配置。"""
+    """读取草稿生成配置。"""
     raw = _read_config().get("draft", {})
     if not isinstance(raw, dict):
         raw = {}
 
     return DraftConfig(
-        max_chars=_env_int("KB_DRAFT_MAX_CHARS", raw.get("max_chars"), 3600),
-        context_chars=_env_int("KB_DRAFT_CONTEXT_CHARS", raw.get("context_chars"), 800),
-        outline_max_sections=_env_int("KB_DRAFT_OUTLINE_MAX_SECTIONS", raw.get("outline_max_sections"), 40),
+        max_chars=_as_int(raw.get("max_chars"), 3600),
+        context_chars=_as_int(raw.get("context_chars"), 800),
+        outline_max_sections=_as_int(raw.get("outline_max_sections"), 40),
     )
 
 
 def _read_config() -> Dict[str, Any]:
     """读取 YAML 配置文件并返回字典。"""
-    path = Path(os.environ.get("KB_INGEST_CONFIG", DEFAULT_CONFIG_PATH))
-    if not path.exists():
+    if not DEFAULT_CONFIG_PATH.exists():
         return {}
-    data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+    data = yaml.safe_load(DEFAULT_CONFIG_PATH.read_text(encoding="utf-8")) or {}
     return data if isinstance(data, dict) else {}
 
 
-def _env_bool(name: str, default: bool) -> bool:
-    """读取布尔环境变量并回退到默认值。"""
-    value = os.environ.get(name)
-    if value is None:
-        return default
-    return _as_bool(value, default)
-
-
-def _env_int(name: str, value: Any, default: int) -> int:
-    """读取整数环境变量并回退到默认值。"""
-    raw = os.environ.get(name, value)
+def _as_int(value: Any, default: int) -> int:
+    """把配置值转换为整数。"""
     try:
-        return int(raw)
+        return int(value)
     except (TypeError, ValueError):
         return default
 
 
-def _env_float(name: str, value: Any, default: float) -> float:
-    """读取浮点环境变量并回退到默认值。"""
-    raw = os.environ.get(name, value)
+def _as_float(value: Any, default: float) -> float:
+    """把配置值转换为浮点数。"""
     try:
-        return float(raw)
+        return float(value)
     except (TypeError, ValueError):
         return default
 

package/ingest.py

@@ -2,17 +2,22 @@
 from __future__ import annotations
 
 import argparse
+import json
 import sys
 from dataclasses import replace
+from datetime import datetime
 from pathlib import Path
 from typing import List
+from uuid import uuid4
+
+import yaml
 
 CURRENT_DIR = Path(__file__).resolve().parent
 if str(CURRENT_DIR) not in sys.path:
     sys.path.insert(0, str(CURRENT_DIR))
 
 from app_config import get_draft_config
-from normalizer import normalize_block
+from normalizer import fallback_failed_block, normalize_block
 from parser import iter_input_files, parse_document
 from schemas import ParsedBlock
 from splitter import split_blocks
@@ -21,6 +26,7 @@ from writer import write_item
 
 
 IGNORED_EXISTING_FILES = {".gitkeep", ".DS_Store"}
+PROGRESS_FILENAME = ".draft_progress.json"
 
 
 def cmd_parse(args) -> int:
@@ -45,21 +51,46 @@ def cmd_draft(args) -> int:
     output_dir = Path(args.output)
 
     existing = _list_effective_files(output_dir)
-    if existing and not _confirm_overwrite(output_dir, existing):
-        print("aborted. existing files were kept.")
-        return 0
-
+    progress_path = output_dir / PROGRESS_FILENAME
+    resume_state = None
     if existing:
-        _clear_generated_files(output_dir)
+        action = _choose_existing_result_action(output_dir, existing)
+        if action == "exit":
+            print("aborted. existing files were kept.")
+            return 0
+        if action == "rebuild":
+            _clear_generated_files(output_dir)
+        elif action == "retry":
+            return _retry_failed_files(output_dir, status=args.status)
+        elif action == "resume":
+            resume_state = _load_progress(progress_path)
+            if not resume_state:
+                print(f"aborted. no usable checkpoint found at {progress_path}.")
+                return 1
 
     output_dir.mkdir(parents=True, exist_ok=True)
 
-    total_items = 0
-    source_order = 0
+    run_timestamp = (
+        str(resume_state.get("run_timestamp"))
+        if resume_state
+        else _make_timestamp()
+    )
+    run_trace_id = (
+        str(resume_state.get("run_trace_id"))
+        if resume_state
+        else uuid4().hex[:8]
+    )
+    total_items = int(resume_state.get("total_items", 0)) if resume_state else 0
+    source_order = int(resume_state.get("source_order", 0)) if resume_state else 0
     draft_config = get_draft_config()
     max_chars = args.max_chars or draft_config.max_chars
     files = iter_input_files(input_path)
-    for path in files:
+    start_file_index = int(resume_state.get("file_index", 0)) if resume_state else 0
+    start_block_index = int(resume_state.get("block_index", 0)) if resume_state else 0
+
+    for file_index, path in enumerate(files):
+        if file_index < start_file_index:
+            continue
         parsed = parse_document(path)
         blocks = split_blocks(parsed.blocks, max_chars=max_chars)
         blocks = _attach_block_context(
@@ -67,15 +98,59 @@ def cmd_draft(args) -> int:
             context_chars=draft_config.context_chars,
             outline_max_sections=draft_config.outline_max_sections,
         )
-        for block in blocks:
-            for item in normalize_block(block, status=args.status):
-                source_order += 1
-                item.source_order = source_order
-                item.source_pages = sorted(set(block.pages))
-                item.source_trace = _source_trace(block)
-                write_item(item, output_dir)
-                total_items += 1
+        block_start = start_block_index if file_index == start_file_index else 0
+        for block_index, block in enumerate(blocks):
+            if block_index < block_start:
+                continue
+            _save_progress(
+                progress_path,
+                input_path=input_path,
+                output_dir=output_dir,
+                files=files,
+                run_timestamp=run_timestamp,
+                run_trace_id=run_trace_id,
+                source_order=source_order,
+                total_items=total_items,
+                file_index=file_index,
+                block_index=block_index,
+                status="running",
+            )
+            try:
+                items = normalize_block(block, status=args.status)
+            except SystemExit as exc:
+                print(f"WARNING: block failed with SystemExit({exc.code}); writing failed fallback")
+                items = fallback_failed_block(block, status=args.status)
+            except Exception as exc:
+                print(f"WARNING: block failed with {type(exc).__name__}: {exc}; writing failed fallback")
+                items = fallback_failed_block(block, status=args.status)
+            if not items:
+                items = fallback_failed_block(block, status=args.status)
+            source_order, written = _write_items(
+                items,
+                output_dir,
+                block,
+                source_order=source_order,
+                run_timestamp=run_timestamp,
+                run_trace_id=run_trace_id,
+            )
+            total_items += written
+            _save_progress(
+                progress_path,
+                input_path=input_path,
+                output_dir=output_dir,
+                files=files,
+                run_timestamp=run_timestamp,
+                run_trace_id=run_trace_id,
+                source_order=source_order,
+                total_items=total_items,
+                file_index=file_index,
+                block_index=block_index + 1,
+                status="running",
+            )
         print(f"drafted: {path} blocks={len(blocks)}")
+        start_block_index = 0
+    if progress_path.exists():
+        progress_path.unlink()
     print(f"done. files={len(files)} draft_items={total_items} output={output_dir}")
     return 0
 
@@ -90,16 +165,205 @@ def _list_effective_files(path: Path) -> list[Path]:
     )
 
 
-def _confirm_overwrite(
+def _write_items(
+    items,
     output_dir: Path,
-    existing: list[Path],
-) -> bool:
-    """询问用户是否覆盖已有生成文件。"""
+    block: ParsedBlock,
+    *,
+    source_order: int,
+    run_timestamp: str,
+    run_trace_id: str,
+) -> tuple[int, int]:
+    written = 0
+    for item in items:
+        source_order += 1
+        item.source_order = source_order
+        item.source_pages = sorted(set(block.pages))
+        item.source_trace = _source_trace(block)
+        write_item(
+            item,
+            output_dir,
+            source_title=Path(block.source_doc).stem,
+            timestamp=run_timestamp,
+            trace_id=run_trace_id,
+        )
+        written += 1
+    return source_order, written
+
+
+def _retry_failed_files(output_dir: Path, status: str) -> int:
+    failed_files = [path for path in _list_effective_files(output_dir) if "failed" in path.stem.lower()]
+    if not failed_files:
+        print("done. failed_files=0 retried=0")
+        return 0
+
+    run_timestamp = _make_timestamp()
+    run_trace_id = uuid4().hex[:8]
+    retried = 0
+    succeeded = 0
+    still_failed = 0
+    for path in failed_files:
+        block = _block_from_failed_file(path)
+        if not block:
+            print(f"WARNING: skipped failed file without chunk source: {path}")
+            continue
+        try:
+            items = normalize_block(block, status=status)
+        except SystemExit as exc:
+            print(f"WARNING: retry failed with SystemExit({exc.code}); keeping failed fallback: {path}")
+            items = fallback_failed_block(block, status=status)
+        except Exception as exc:
+            print(f"WARNING: retry failed with {type(exc).__name__}: {exc}; keeping failed fallback: {path}")
+            items = fallback_failed_block(block, status=status)
+        if not items:
+            items = fallback_failed_block(block, status=status)
+
+        source_order = max(int(block.order or 0) - 1, 0)
+        _, written = _write_items(
+            items,
+            output_dir,
+            block,
+            source_order=source_order,
+            run_timestamp=run_timestamp,
+            run_trace_id=run_trace_id,
+        )
+        if written:
+            path.unlink()
+        retried += 1
+        if any(item.review_status == "failed" for item in items):
+            still_failed += 1
+        else:
+            succeeded += 1
+    print(f"done. failed_files={len(failed_files)} retried={retried} succeeded={succeeded} still_failed={still_failed}")
+    return 0
+
+
+def _block_from_failed_file(path: Path) -> ParsedBlock | None:
+    text = path.read_text(encoding="utf-8")
+    metadata = _front_matter(text)
+    source = _failed_chunk_source(text)
+    if not source:
+        return None
+    return ParsedBlock(
+        source_doc=str(metadata.get("source_doc") or path.stem),
+        source_section=str(metadata.get("source_section") or ""),
+        content=source,
+        pages=[int(page) for page in metadata.get("source_pages") or [] if str(page).isdigit()],
+        order=int(metadata.get("source_order") or 0),
+        category=str(metadata.get("category") or ""),
+        category_keywords=[str(item) for item in metadata.get("category_keywords") or []],
+        source_doc_description=str(metadata.get("source_doc_description") or ""),
+        subcategory=str(metadata.get("subcategory") or ""),
+        category_path=[str(item) for item in metadata.get("category_path") or []],
+        related_categories=[str(item) for item in metadata.get("related_categories") or []],
+        relation_notes=[str(item) for item in metadata.get("relation_notes") or []],
+        related_items=metadata.get("related_items") or [],
+    )
+
+
+def _front_matter(text: str) -> dict:
+    if not text.startswith("---\n"):
+        return {}
+    end = text.find("\n---\n", 4)
+    if end < 0:
+        return {}
+    data = yaml.safe_load(text[4:end]) or {}
+    return data if isinstance(data, dict) else {}
+
+
+def _failed_chunk_source(text: str) -> str:
+    marker = "## failed_chunk_source"
+    start = text.find(marker)
+    if start < 0:
+        return ""
+    source = text[start + len(marker):].strip()
+    if source.startswith("```"):
+        first_line = source.find("\n")
+        if first_line >= 0:
+            source = source[first_line + 1:]
+        if source.endswith("```"):
+            source = source[:-3]
+    return source.strip()
+
+
+def _choose_existing_result_action(output_dir: Path, existing: list[Path]) -> str:
+    """询问用户如何处理已有生成结果。"""
     print(f"found {len(existing)} existing file(s) in {output_dir}.")
-    print("Continuing will delete existing generated files under:")
-    print(f"- {output_dir}")
-    answer = input("Overwrite and continue? [y/N]: ").strip().lower()
-    return answer in {"y", "yes"}
+    print("Choose how to continue:")
+    print("1. delete and rebuild")
+    print("2. resume from checkpoint")
+    print("3. retry failed files")
+    print("4. exit")
+    answer = input("Select [1/2/3/4]: ").strip().lower().translate(
+        str.maketrans({"１": "1", "２": "2", "３": "3", "４": "4"})
+    )
+    if answer.startswith("1") or answer in {"d", "delete", "rebuild", "r"}:
+        return "rebuild"
+    if answer.startswith("2") or answer in {"resume", "continue", "c"}:
+        return "resume"
+    if answer.startswith("3") or answer in {"retry", "failed", "f"}:
+        return "retry"
+    return "exit"
+
+
+def _load_progress(path: Path) -> dict | None:
+    """读取断点续传状态。"""
+    if not path.exists():
+        return None
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        print(f"WARNING: failed to read checkpoint: {exc}")
+        return None
+    if not isinstance(data, dict):
+        return None
+    return data
+
+
+def _save_progress(
+    path: Path,
+    *,
+    input_path: Path,
+    output_dir: Path,
+    files: list[Path],
+    run_timestamp: str,
+    run_trace_id: str,
+    source_order: int,
+    total_items: int,
+    file_index: int,
+    block_index: int,
+    status: str,
+    error: str = "",
+) -> None:
+    """保存 draft 断点续传状态。"""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    current_file = files[file_index] if 0 <= file_index < len(files) else None
+    payload = {
+        "version": 1,
+        "status": status,
+        "error": error,
+        "input_path": str(input_path),
+        "output_dir": str(output_dir),
+        "run_timestamp": run_timestamp,
+        "run_trace_id": run_trace_id,
+        "source_order": source_order,
+        "total_items": total_items,
+        "file_index": file_index,
+        "block_index": block_index,
+        "current_file": str(current_file) if current_file else "",
+        "current_file_name": current_file.name if current_file else "",
+        "files": [str(path) for path in files],
+        "updated_at": _make_timestamp(),
+    }
+    path.write_text(
+        json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
+        encoding="utf-8",
+    )
+
+
+def _make_timestamp() -> str:
+    """生成用于文件名和断点记录的本地时间戳。"""
+    return datetime.now().strftime("%Y%m%d%H%M%S")
 
 
 def _clear_generated_files(*dirs: Path) -> None:

package/input/function/.gitkeep

@@ -0,0 +1 @@
+

package/input/function/tools.yaml

@@ -0,0 +1,27 @@
+tools:
+  - name: "query_member_change_announcements"
+    display_name: "查询成员机构变更通知"
+    description: "查询成员机构近期变更、变更报备、计划变更、执行评价以及影响范围。"
+    trigger: "当用户询问成员机构变更通知、变更报备、计划变更或执行评价时优先调用。"
+    input_schema:
+      type: "object"
+      properties:
+        orgCode:
+          type: "string"
+          description: "成员机构金融编码，通常由 resolve_member_org 工具获得。"
+      required: ["orgCode"]
+    output_schema:
+      type: "array"
+      description: "匹配到的成员机构变更通知列表。"
+      items:
+        type: "object"
+        properties:
+          member_code:
+            type: "string"
+            description: "成员机构编码。"
+          member_name:
+            type: "string"
+            description: "成员机构名称。"
+          change_type:
+            type: "string"
+            description: "变更类型。"