union_kb_ingest 1.0.8 → 1.0.10

package/README.md

@@ -43,7 +43,7 @@ input/
 python ingest.py draft
 ```
 
-如果 `result/` 中已有生成文件，命令会先询问是否覆盖。选择 `y` 后会清空 `result/` 中已有生成文件，再重新生成；选择其他内容会直接退出，避免多次生成结果相互影响。
+如果 `result/` 中已有生成文件，命令会提示选择删除重建、从断点继续或退出。断点状态保存在 `result/.draft_progress.json`，大模型多次重试失败退出时会记录当前文件和片段位置，下次可选择从断点继续。
 
 只解析为中间 Markdown：
 

package/config/config.yaml

@@ -3,7 +3,7 @@ llm:
   timeout_seconds: 120
   max_tokens: 8192
   temperature: 0.1
-  api_key: "15f066c4509845038027ea5746524af5.w4CLSC6ODiKVC1wK"
+  api_key: ""
   model: "GLM-4.7-Flash"
   base_url: "https://open.bigmodel.cn/api/paas/v4/"
 

package/ingest.py

@@ -2,10 +2,13 @@
 from __future__ import annotations
 
 import argparse
+import json
 import sys
 from dataclasses import replace
+from datetime import datetime
 from pathlib import Path
 from typing import List
+from uuid import uuid4
 
 CURRENT_DIR = Path(__file__).resolve().parent
 if str(CURRENT_DIR) not in sys.path:
@@ -21,6 +24,7 @@ from writer import write_item
 
 
 IGNORED_EXISTING_FILES = {".gitkeep", ".DS_Store"}
+PROGRESS_FILENAME = ".draft_progress.json"
 
 
 def cmd_parse(args) -> int:
@@ -45,21 +49,44 @@ def cmd_draft(args) -> int:
     output_dir = Path(args.output)
 
     existing = _list_effective_files(output_dir)
-    if existing and not _confirm_overwrite(output_dir, existing):
-        print("aborted. existing files were kept.")
-        return 0
-
+    progress_path = output_dir / PROGRESS_FILENAME
+    resume_state = None
     if existing:
-        _clear_generated_files(output_dir)
+        action = _choose_existing_result_action(output_dir, existing)
+        if action == "exit":
+            print("aborted. existing files were kept.")
+            return 0
+        if action == "rebuild":
+            _clear_generated_files(output_dir)
+        elif action == "resume":
+            resume_state = _load_progress(progress_path)
+            if not resume_state:
+                print(f"aborted. no usable checkpoint found at {progress_path}.")
+                return 1
 
     output_dir.mkdir(parents=True, exist_ok=True)
 
-    total_items = 0
-    source_order = 0
+    run_timestamp = (
+        str(resume_state.get("run_timestamp"))
+        if resume_state
+        else _make_timestamp()
+    )
+    run_trace_id = (
+        str(resume_state.get("run_trace_id"))
+        if resume_state
+        else uuid4().hex[:8]
+    )
+    total_items = int(resume_state.get("total_items", 0)) if resume_state else 0
+    source_order = int(resume_state.get("source_order", 0)) if resume_state else 0
     draft_config = get_draft_config()
     max_chars = args.max_chars or draft_config.max_chars
     files = iter_input_files(input_path)
-    for path in files:
+    start_file_index = int(resume_state.get("file_index", 0)) if resume_state else 0
+    start_block_index = int(resume_state.get("block_index", 0)) if resume_state else 0
+
+    for file_index, path in enumerate(files):
+        if file_index < start_file_index:
+            continue
         parsed = parse_document(path)
         blocks = split_blocks(parsed.blocks, max_chars=max_chars)
         blocks = _attach_block_context(
@@ -67,15 +94,76 @@ def cmd_draft(args) -> int:
             context_chars=draft_config.context_chars,
             outline_max_sections=draft_config.outline_max_sections,
         )
-        for block in blocks:
-            for item in normalize_block(block, status=args.status):
+        block_start = start_block_index if file_index == start_file_index else 0
+        for block_index, block in enumerate(blocks):
+            if block_index < block_start:
+                continue
+            _save_progress(
+                progress_path,
+                input_path=input_path,
+                output_dir=output_dir,
+                files=files,
+                run_timestamp=run_timestamp,
+                run_trace_id=run_trace_id,
+                source_order=source_order,
+                total_items=total_items,
+                file_index=file_index,
+                block_index=block_index,
+                status="running",
+            )
+            try:
+                items = normalize_block(block, status=args.status)
+            except SystemExit as exc:
+                _save_progress(
+                    progress_path,
+                    input_path=input_path,
+                    output_dir=output_dir,
+                    files=files,
+                    run_timestamp=run_timestamp,
+                    run_trace_id=run_trace_id,
+                    source_order=source_order,
+                    total_items=total_items,
+                    file_index=file_index,
+                    block_index=block_index,
+                    status="failed",
+                    error=f"SystemExit({exc.code})",
+                )
+                print(
+                    "checkpoint saved. "
+                    f"file={path.name} block={block_index + 1}/{len(blocks)} "
+                    f"progress={progress_path}"
+                )
+                raise
+            for item in items:
                 source_order += 1
                 item.source_order = source_order
                 item.source_pages = sorted(set(block.pages))
                 item.source_trace = _source_trace(block)
-                write_item(item, output_dir)
+                write_item(
+                    item,
+                    output_dir,
+                    source_title=Path(block.source_doc).stem,
+                    timestamp=run_timestamp,
+                    trace_id=run_trace_id,
+                )
                 total_items += 1
+            _save_progress(
+                progress_path,
+                input_path=input_path,
+                output_dir=output_dir,
+                files=files,
+                run_timestamp=run_timestamp,
+                run_trace_id=run_trace_id,
+                source_order=source_order,
+                total_items=total_items,
+                file_index=file_index,
+                block_index=block_index + 1,
+                status="running",
+            )
         print(f"drafted: {path} blocks={len(blocks)}")
+        start_block_index = 0
+    if progress_path.exists():
+        progress_path.unlink()
     print(f"done. files={len(files)} draft_items={total_items} output={output_dir}")
     return 0
 
@@ -90,16 +178,81 @@ def _list_effective_files(path: Path) -> list[Path]:
     )
 
 
-def _confirm_overwrite(
-    output_dir: Path,
-    existing: list[Path],
-) -> bool:
-    """询问用户是否覆盖已有生成文件。"""
+def _choose_existing_result_action(output_dir: Path, existing: list[Path]) -> str:
+    """询问用户如何处理已有生成结果。"""
     print(f"found {len(existing)} existing file(s) in {output_dir}.")
-    print("Continuing will delete existing generated files under:")
-    print(f"- {output_dir}")
-    answer = input("Overwrite and continue? [y/N]: ").strip().lower()
-    return answer in {"y", "yes"}
+    print("Choose how to continue:")
+    print("1. delete and rebuild")
+    print("2. resume from checkpoint")
+    print("3. exit")
+    answer = input("Select [1/2/3]: ").strip().lower().translate(
+        str.maketrans({"１": "1", "２": "2", "３": "3"})
+    )
+    if answer.startswith("1") or answer in {"d", "delete", "rebuild", "r"}:
+        return "rebuild"
+    if answer.startswith("2") or answer in {"resume", "continue", "c"}:
+        return "resume"
+    return "exit"
+
+
+def _load_progress(path: Path) -> dict | None:
+    """读取断点续传状态。"""
+    if not path.exists():
+        return None
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        print(f"WARNING: failed to read checkpoint: {exc}")
+        return None
+    if not isinstance(data, dict):
+        return None
+    return data
+
+
+def _save_progress(
+    path: Path,
+    *,
+    input_path: Path,
+    output_dir: Path,
+    files: list[Path],
+    run_timestamp: str,
+    run_trace_id: str,
+    source_order: int,
+    total_items: int,
+    file_index: int,
+    block_index: int,
+    status: str,
+    error: str = "",
+) -> None:
+    """保存 draft 断点续传状态。"""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    current_file = files[file_index] if 0 <= file_index < len(files) else None
+    payload = {
+        "version": 1,
+        "status": status,
+        "error": error,
+        "input_path": str(input_path),
+        "output_dir": str(output_dir),
+        "run_timestamp": run_timestamp,
+        "run_trace_id": run_trace_id,
+        "source_order": source_order,
+        "total_items": total_items,
+        "file_index": file_index,
+        "block_index": block_index,
+        "current_file": str(current_file) if current_file else "",
+        "current_file_name": current_file.name if current_file else "",
+        "files": [str(path) for path in files],
+        "updated_at": _make_timestamp(),
+    }
+    path.write_text(
+        json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
+        encoding="utf-8",
+    )
+
+
+def _make_timestamp() -> str:
+    """生成用于文件名和断点记录的本地时间戳。"""
+    return datetime.now().strftime("%Y%m%d%H%M%S")
 
 
 def _clear_generated_files(*dirs: Path) -> None:

package/normalizer.py

@@ -9,6 +9,8 @@ from functools import lru_cache
 from pathlib import Path
 from typing import Any, Dict, List
 
+import yaml
+
 from app_config import get_llm_config
 from schemas import DOC_TYPES, KnowledgeItem, ParsedBlock
 
@@ -17,7 +19,7 @@ DEFAULT_DOMAIN = "网联清算业务"
 DEFAULT_OWNER = "网联清算业务知识库"
 CURRENT_DIR = Path(__file__).resolve().parent
 KB_SPEC_PATH = CURRENT_DIR / "prompts" / "知识库建立规范.md"
-TOOLS_PATH = CURRENT_DIR / "input" / "function" / "tools.json"
+TOOLS_PATH = CURRENT_DIR / "input" / "function" / "tools.yaml"
 LLM_MAX_RETRIES = 10
 COVERAGE_MAX_RETRIES = 3
 
@@ -743,9 +745,11 @@ def _read_kb_spec() -> str:
 def _read_tools() -> List[Dict[str, Any]]:
     """读取本地工具维护文件。"""
     try:
-        raw = json.loads(TOOLS_PATH.read_text(encoding="utf-8"))
-    except (FileNotFoundError, json.JSONDecodeError):
+        raw = yaml.safe_load(TOOLS_PATH.read_text(encoding="utf-8"))
+    except (FileNotFoundError, yaml.YAMLError):
         return []
+    if isinstance(raw, dict):
+        raw = raw.get("tools")
     if not isinstance(raw, list):
         return []
     return [item for item in raw if isinstance(item, dict)]

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "union_kb_ingest",
-  "version": "1.0.8",
+  "version": "1.0.10",
   "description": "Offline knowledge-base ingest helper for PDF, Word, Markdown and TXT documents.",
   "bin": {
     "union_kb_ingest": "bin/union_kb_ingest"

package/parser.py

@@ -21,7 +21,7 @@ def iter_input_files(input_path: Path) -> List[Path]:
         p for p in input_path.rglob("*")
         if p.is_file() and p.suffix.lower() in SUPPORTED_EXTENSIONS
     ]
-    return sorted(files)
+    return sorted(files, key=lambda path: (path.name.lower(), str(path).lower()))
 
 
 def parse_document(path: Path) -> ParsedDocument:

package/writer.py

@@ -8,10 +8,25 @@ import yaml
 from schemas import KnowledgeItem
 
 
-def write_item(item: KnowledgeItem, output_dir: Path) -> Path:
+def write_item(
+    item: KnowledgeItem,
+    output_dir: Path,
+    *,
+    source_title: str | None = None,
+    timestamp: str | None = None,
+    trace_id: str | None = None,
+) -> Path:
     """把知识条目渲染并写入输出目录。"""
     output_dir.mkdir(parents=True, exist_ok=True)
-    path = output_dir / f"{_order_prefix(item.source_order)}-{_safe_filename(item.kb_id)}.md"
+    parts = [
+        source_title,
+        timestamp,
+        trace_id,
+        _order_prefix(item.source_order),
+        item.kb_id,
+    ]
+    filename = "-".join(_safe_filename(part) for part in parts if part)
+    path = output_dir / f"{filename}.md"
     path.write_text(render_markdown(item), encoding="utf-8")
     return path
 
@@ -30,7 +45,7 @@ def render_markdown(item: KnowledgeItem) -> str:
 def _safe_filename(value: str) -> str:
     """把标识符转换为安全文件名。"""
     value = value.lower().strip()
-    value = re.sub(r"[^a-z0-9._-]+", "-", value)
+    value = re.sub(r"[^\w._-]+", "-", value)
     value = re.sub(r"-+", "-", value).strip("-")
     return value or "kb-item"