union_kb_ingest 1.0.0

package/splitter.py

@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+import re
+from typing import List
+
+from schemas import ParsedBlock
+
+
+def split_blocks(blocks: List[ParsedBlock], max_chars: int = 8000) -> List[ParsedBlock]:
+    result: List[ParsedBlock] = []
+    for block in _merge_parent_child_blocks(blocks):
+        result.extend(_split_one(block, max_chars=max_chars))
+    return [b for b in result if len(b.content.strip()) >= 40]
+
+
+def _merge_parent_child_blocks(blocks: List[ParsedBlock]) -> List[ParsedBlock]:
+    merged: List[ParsedBlock] = []
+    current: ParsedBlock | None = None
+    current_prefix = ""
+
+    for block in blocks:
+        prefix = _section_prefix(block.source_section)
+        if prefix and _is_parent_section(prefix, block.content):
+            if current:
+                merged.append(current)
+            current = block
+            current_prefix = prefix
+            continue
+
+        if current and current_prefix and _is_child_section(current_prefix, block.source_section):
+            current = ParsedBlock(
+                source_doc=current.source_doc,
+                source_section=current.source_section,
+                content=f"{current.content.rstrip()}\n\n{block.content.strip()}",
+                pages=sorted(set(current.pages + block.pages)),
+                order=current.order,
+            )
+            continue
+
+        if current:
+            merged.append(current)
+            current = None
+            current_prefix = ""
+        merged.append(block)
+
+    if current:
+        merged.append(current)
+    return merged
+
+
+def _section_prefix(title: str) -> str:
+    match = re.match(r"^(\d+\.\d+)(?!\.)", title.strip())
+    return match.group(1) if match else ""
+
+
+def _is_parent_section(prefix: str, content: str) -> bool:
+    return bool(prefix) and len(content.strip()) < 120
+
+
+def _is_child_section(parent_prefix: str, title: str) -> bool:
+    return title.strip().startswith(f"{parent_prefix}.")
+
+
+def _split_one(block: ParsedBlock, max_chars: int) -> List[ParsedBlock]:
+    content = block.content.strip()
+    if len(content) <= max_chars:
+        return [block]
+
+    candidates = _split_by_heading_window(content, max_chars)
+
+    output: List[ParsedBlock] = []
+    for idx, text in enumerate(candidates, start=1):
+        text = text.strip()
+        if not text:
+            continue
+        output.append(ParsedBlock(
+            source_doc=block.source_doc,
+            source_section=f"{block.source_section} / 片段 {idx}",
+            content=text,
+            pages=block.pages,
+            order=block.order * 100 + idx,
+        ))
+    return output
+
+
+def _split_by_heading_window(text: str, max_chars: int) -> List[str]:
+    sections = [p.strip() for p in re.split(r"\n(?=#{2,4}\s+)", text) if p.strip()]
+    if len(sections) <= 1:
+        return _split_by_paragraph_window(text, max_chars)
+
+    chunks: List[str] = []
+    current: List[str] = []
+    current_len = 0
+    for section in sections:
+        if current and current_len + len(section) > max_chars:
+            chunks.append("\n\n".join(current))
+            current = []
+            current_len = 0
+        if len(section) > max_chars:
+            if current:
+                chunks.append("\n\n".join(current))
+                current = []
+                current_len = 0
+            chunks.extend(_split_by_paragraph_window(section, max_chars))
+            continue
+        current.append(section)
+        current_len += len(section)
+    if current:
+        chunks.append("\n\n".join(current))
+    return chunks or [text]
+
+
+def _split_by_paragraph_window(text: str, max_chars: int) -> List[str]:
+    paragraphs = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
+    chunks: List[str] = []
+    current: List[str] = []
+    current_len = 0
+    for paragraph in paragraphs:
+        if current and current_len + len(paragraph) > max_chars:
+            chunks.append("\n\n".join(current))
+            current = []
+            current_len = 0
+        current.append(paragraph)
+        current_len += len(paragraph)
+    if current:
+        chunks.append("\n\n".join(current))
+    return chunks or [text]

package/validator.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Dict, List, Tuple
+
+import yaml
+
+from schemas import DOC_TYPES, ValidationIssue
+
+
+REQUIRED_FIELDS = [
+    "kb_id",
+    "title",
+    "doc_type",
+    "domain",
+    "business_modules",
+    "source_doc",
+    "source_version",
+    "source_section",
+    "owner",
+    "confidentiality",
+    "risk_level",
+    "applicable_roles",
+    "tags",
+    "status",
+]
+
+
+def validate_dir(input_dir: Path) -> List[ValidationIssue]:
+    issues: List[ValidationIssue] = []
+    for path in sorted(input_dir.rglob("*.md")):
+        issues.extend(validate_file(path))
+    return issues
+
+
+def validate_file(path: Path) -> List[ValidationIssue]:
+    issues: List[ValidationIssue] = []
+    try:
+        metadata, body = _read_markdown(path)
+    except Exception as exc:
+        return [ValidationIssue(path, "error", f"无法读取 Markdown Front Matter: {exc}")]
+
+    for field in REQUIRED_FIELDS:
+        if field not in metadata:
+            issues.append(ValidationIssue(path, "error", f"缺少必填字段: {field}"))
+
+    if metadata.get("doc_type") not in DOC_TYPES:
+        issues.append(ValidationIssue(path, "error", f"doc_type 不合法: {metadata.get('doc_type')}"))
+
+    if metadata.get("status") == "active" and metadata.get("review_status") != "approved":
+        issues.append(ValidationIssue(path, "warning", "active 条目建议先设置 review_status: approved"))
+
+    if len(body) > 3000:
+        issues.append(ValidationIssue(path, "warning", f"正文较长，建议继续拆分: {len(body)} 字符"))
+    if len(body) < 200:
+        issues.append(ValidationIssue(path, "warning", f"正文较短，可能信息不足: {len(body)} 字符"))
+
+    headings = re.findall(r"^##\s+", body, re.M)
+    if len(headings) < 3:
+        issues.append(ValidationIssue(path, "warning", "二级标题过少，可能未遵照标准结构"))
+
+    return issues
+
+
+def _read_markdown(path: Path) -> Tuple[Dict, str]:
+    text = path.read_text(encoding="utf-8")
+    match = re.match(r"^---\n(.*?)\n---\n(.*)$", text, re.S)
+    if not match:
+        raise ValueError("缺少 YAML Front Matter")
+    return yaml.safe_load(match.group(1)) or {}, match.group(2).strip()
+

package/writer.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import yaml
+
+from schemas import KnowledgeItem
+
+
+def write_item(item: KnowledgeItem, output_dir: Path) -> Path:
+    output_dir.mkdir(parents=True, exist_ok=True)
+    path = output_dir / f"{_safe_filename(item.kb_id)}.md"
+    path.write_text(render_markdown(item), encoding="utf-8")
+    return path
+
+
+def render_markdown(item: KnowledgeItem) -> str:
+    metadata = yaml.safe_dump(
+        item.metadata(),
+        allow_unicode=True,
+        sort_keys=False,
+        default_flow_style=False,
+    ).strip()
+    return f"---\n{metadata}\n---\n\n{item.body.strip()}\n"
+
+
+def _safe_filename(value: str) -> str:
+    value = value.lower().strip()
+    value = re.sub(r"[^a-z0-9._-]+", "-", value)
+    value = re.sub(r"-+", "-", value).strip("-")
+    return value or "kb-item"
+