source-kb 0.2.2__py3-none-any.whl

core/validators/links.py

@@ -0,0 +1,76 @@
+"""Link integrity validator — dead link and anchor detection.
+
+Checks all markdown links in documents:
+- File links: [text](./file.md) → verify file exists
+- Anchor links: [text](./file.md#heading) → verify heading exists in target
+
+Usage:
+    from core.validators.links import LinksValidator
+    result = LinksValidator().validate(module_dir)
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Any
+
+from core.interfaces import Validator, ValidationResult
+from core.validators import register_validator
+
+LINK_PATTERN = re.compile(r'\[([^\]]*)\]\((\./[^)]+)\)')
+
+
+@register_validator
+class LinksValidator(Validator):
+    """Check markdown link integrity."""
+
+    name = "links"
+
+    def validate(self, module_dir: Path, **kwargs: Any) -> ValidationResult:
+        result = ValidationResult()
+
+        for md_file in sorted(module_dir.glob("*.md")):
+            if md_file.name.startswith("."):
+                continue
+            content = md_file.read_text(encoding="utf-8")
+
+            for match in LINK_PATTERN.finditer(content):
+                link_text = match.group(1)
+                link_target = match.group(2)
+
+                # Split path and anchor
+                if "#" in link_target:
+                    file_part, anchor = link_target.rsplit("#", 1)
+                else:
+                    file_part, anchor = link_target, None
+
+                # Resolve target file
+                target = (md_file.parent / file_part).resolve()
+                if not target.exists():
+                    result.errors.append(
+                        f"links: {md_file.name} → dead link '{link_target}' (file not found)"
+                    )
+                    continue
+
+                # Check anchor if present
+                if anchor and target.suffix == ".md":
+                    target_content = target.read_text(encoding="utf-8")
+                    if not _has_heading(target_content, anchor):
+                        result.warnings.append(
+                            f"links: {md_file.name} → anchor '#{anchor}' not found in {target.name}"
+                        )
+
+        return result
+
+
+def _has_heading(content: str, anchor: str) -> bool:
+    """Check if a document contains a heading matching the anchor slug."""
+    anchor_lower = anchor.lower()
+    for line in content.splitlines():
+        if line.startswith("#"):
+            heading_text = line.lstrip("#").strip()
+            slug = re.sub(r'[^\w\u4e00-\u9fff-]', '-', heading_text).strip('-').lower()
+            if slug == anchor_lower:
+                return True
+    return False

core/validators/sampling.py

@@ -0,0 +1,169 @@
+"""Numeric sampling validator — spot-check values from skeleton in docs.
+
+Validates that skeleton-derived values (enum names, field names) actually appear
+in generated documents. Catches fabricated content.
+
+The actual sampling logic is now in core/validators/engine.py (sampling_check handler).
+This module provides the SamplingValidator class for the registry and the optional
+LLM-enhanced semantic check.
+
+Usage:
+    from core.validators.sampling import SamplingValidator
+    result = SamplingValidator().validate(module_dir, preset=preset)
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import random
+from pathlib import Path
+from typing import Any
+
+from core.interfaces import Validator, ValidationResult
+from core.validators import register_validator
+
+logger = logging.getLogger(__name__)
+
+
+@register_validator
+class SamplingValidator(Validator):
+    """Spot-check skeleton values against generated documents.
+
+    Delegates to the validator engine for preset-configured sampling checks.
+    Falls back to generic field_names sampling if no validators configured.
+    """
+
+    name = "sampling"
+
+    def validate(self, module_dir: Path, **kwargs: Any) -> ValidationResult:
+        result = ValidationResult()
+        preset = kwargs.get("preset", {})
+
+        from core.paths import resolve_skeleton_summary
+        summary_file = resolve_skeleton_summary(module_dir)
+        if not summary_file:
+            return result
+
+        try:
+            entries = json.loads(summary_file.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError):
+            return result
+
+        # Delegate to engine for preset-configured validators
+        from core.validators.engine import run_doc_validators
+        doc_types = preset.get("doc_types", {})
+        for dt_key, dt_config in doc_types.items():
+            if not isinstance(dt_config, dict):
+                continue
+            validators = dt_config.get("validators", [])
+            sampling_validators = [v for v in validators if v.get("type") == "sampling_check"]
+            if not sampling_validators:
+                continue
+            filename = dt_config.get("filename", f"{dt_key}.md")
+            doc_path = module_dir / filename
+            if doc_path.exists():
+                engine_result = run_doc_validators(doc_path, dt_key, preset, entries)
+                result = result.merge(engine_result)
+
+        return result
+
+
+# ---------------------------------------------------------------------------
+# LLM-enhanced semantic validation (optional)
+# ---------------------------------------------------------------------------
+
+
+def llm_semantic_check(
+    module_dir: Path,
+    entries: list[dict],
+    preset: dict | None = None,
+    strategy: Any = None,
+    sample_count: int = 3,
+) -> list[str]:
+    """Use LLM to verify that document descriptions match source code reality.
+
+    Picks a few enum values or field names, extracts their description from docs,
+    and asks LLM to verify accuracy against skeleton data.
+
+    Args:
+        module_dir: Module documentation directory
+        entries: Skeleton entries
+        preset: Preset config (for resolving doc filenames)
+        strategy: LlmStrategy instance (None = skip)
+        sample_count: Number of items to verify
+
+    Returns:
+        List of warning strings for inaccurate descriptions.
+        Empty if no LLM available or all descriptions are accurate.
+    """
+    if strategy is None:
+        return []
+
+    # Sample enum values with their skeleton context
+    enum_samples = []
+    for entry in entries:
+        for cls in entry.get("classes", []):
+            if cls.get("type", "").lower() == "enum":
+                for m in entry.get("methods", []):
+                    if m.get("name", "").isupper():
+                        enum_samples.append({
+                            "name": m["name"],
+                            "file": entry.get("file", ""),
+                            "class": cls.get("name", ""),
+                        })
+
+    if not enum_samples:
+        return []
+
+    samples = random.sample(enum_samples, min(sample_count, len(enum_samples)))
+
+    # Resolve enum doc filename from preset
+    enum_filename = "enums-and-constants.md"
+    if preset:
+        try:
+            from core.preset import get_doc_filename
+            fn = get_doc_filename(preset, "enums-and-constants", strict=False)
+            if fn:
+                enum_filename = fn
+        except Exception:
+            pass
+
+    enum_doc = module_dir / enum_filename
+    if not enum_doc.exists():
+        return []
+    doc_content = enum_doc.read_text(encoding="utf-8")
+
+    # Build verification prompt
+    checks = []
+    for s in samples:
+        idx = doc_content.find(s["name"])
+        if idx < 0:
+            checks.append(f"- `{s['class']}.{s['name']}`: not found in doc")
+            continue
+        context = doc_content[max(0, idx-200):idx+200]
+        checks.append(f"- `{s['class']}.{s['name']}` (from {s['file']})\n  Doc excerpt: {context[:150]}")
+
+    if not checks:
+        return []
+
+    from core.interfaces import LlmRequest
+    try:
+        resp = strategy.call(LlmRequest(
+            system=(
+                "You are a documentation accuracy reviewer. "
+                "Check whether the following enum values are accurately described in the document. "
+                "Only output items with issues, format: `enum_name: issue description`. "
+                "If all are accurate, output 'all accurate'."
+            ),
+            user="\n".join(checks),
+            max_tokens=500,
+            temperature=0.1,
+        ))
+    except Exception:
+        return []
+
+    if resp.status != "done" or not resp.content or "all accurate" in resp.content.lower():
+        return []
+
+    return [line.strip() for line in resp.content.strip().splitlines() if line.strip()]

core/validators/structure.py

@@ -0,0 +1,144 @@
+"""Structure validator — document heading and section checks.
+
+Validates document structure: heading levels, required sections, format rules.
+Migrated from engine/skeleton/validate_output.py.
+
+Usage:
+    from core.validators.structure import StructureValidator
+    result = StructureValidator().validate(module_dir, preset=preset)
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Any
+
+from core.interfaces import Validator, ValidationResult
+from core.validators import register_validator
+
+# ---------------------------------------------------------------------------
+# Default section requirements (overridden by preset section_checks)
+# ---------------------------------------------------------------------------
+
+
+@register_validator
+class StructureValidator(Validator):
+    """Validate document heading structure and required sections."""
+
+    name = "structure"
+
+    def validate(self, module_dir: Path, **kwargs: Any) -> ValidationResult:
+        result = ValidationResult()
+        preset = kwargs.get("preset", {})
+        section_checks = _load_section_checks(preset)
+
+        for md_file in sorted(module_dir.glob("*.md")):
+            if md_file.name.startswith("."):
+                continue
+            headings = extract_headings(md_file)
+            _check_heading_structure(md_file.name, headings, result)
+            _check_required_sections(md_file.name, headings, section_checks, result)
+
+        return result
+
+
+# ---------------------------------------------------------------------------
+# Heading extraction (code-block and front-matter aware)
+# ---------------------------------------------------------------------------
+
+
+def extract_headings(path: Path) -> list[dict]:
+    """Extract markdown headings, skipping code blocks and front-matter.
+
+    Returns list of {level, text, line}.
+    """
+    headings: list[dict] = []
+    try:
+        lines = path.read_text(encoding="utf-8").splitlines()
+    except OSError:
+        return headings
+
+    in_code_block = False
+    in_frontmatter = False
+
+    for i, line in enumerate(lines, 1):
+        stripped = line.strip()
+        # Front-matter detection (only at file start)
+        if i == 1 and stripped == "---":
+            in_frontmatter = True
+            continue
+        if in_frontmatter:
+            if stripped == "---":
+                in_frontmatter = False
+            continue
+        # Code block detection
+        if stripped.startswith("```") or stripped.startswith("~~~"):
+            in_code_block = not in_code_block
+            continue
+        if in_code_block:
+            continue
+        m = re.match(r"^(#{1,4})\s+(.+)", line)
+        if m:
+            headings.append({
+                "level": len(m.group(1)),
+                "text": m.group(2).strip(),
+                "line": i,
+            })
+    return headings
+
+
+# ---------------------------------------------------------------------------
+# Validation helpers
+# ---------------------------------------------------------------------------
+
+
+def _check_heading_structure(
+    fname: str, headings: list[dict], result: ValidationResult
+) -> None:
+    """Check heading level rules: single H1, H1 first, no skipped levels."""
+    if not headings:
+        result.errors.append(f"structure: {fname}: no headings found")
+        return
+
+    if headings[0]["level"] != 1:
+        result.warnings.append(f"structure: {fname}: first heading is not H1")
+
+    h1s = [h for h in headings if h["level"] == 1]
+    if len(h1s) > 1:
+        result.warnings.append(
+            f"structure: {fname}: multiple H1 headings ({len(h1s)}), expected 1"
+        )
+
+
+def _check_required_sections(
+    fname: str,
+    headings: list[dict],
+    section_checks: dict[str, dict],
+    result: ValidationResult,
+) -> None:
+    """Check must_have and should_have section keywords."""
+    rules = section_checks.get(fname)
+    if not rules:
+        return
+
+    h2_texts = " ".join(h["text"] for h in headings if h["level"] <= 2)
+
+    for keyword_spec in rules.get("must_have", []):
+        aliases = keyword_spec if isinstance(keyword_spec, list) else [keyword_spec]
+        if not any(kw in h2_texts for kw in aliases):
+            result.errors.append(
+                f"structure: {fname}: missing required section keyword '{aliases[0]}'"
+            )
+
+    for keyword_spec in rules.get("should_have", []):
+        aliases = keyword_spec if isinstance(keyword_spec, list) else [keyword_spec]
+        if not any(kw in h2_texts for kw in aliases):
+            result.warnings.append(
+                f"structure: {fname}: recommended section keyword '{aliases[0]}' not found"
+            )
+
+
+def _load_section_checks(preset: dict) -> dict[str, dict]:
+    """Load section checks from preset config."""
+    return preset.get("section_checks", {})

engine/__init__.py

@@ -0,0 +1,7 @@
+"""cli — CLI mode execution layer for source-kb.
+
+Assembles core/ tools into executable pipelines with LLM API integration,
+batch execution, retry, heartbeat monitoring, and interactive confirmation.
+
+Entry point: python -m cli.main init --kb my-kb
+"""

engine/assembler.py

@@ -0,0 +1,231 @@
+"""Inline prompt assembler — CLI mode (output-only).
+
+Reads source files and inlines their content into prompts.
+Respects byte limits and priority ordering (high-complexity files first).
+
+This is the CLI-specific PromptAssembler implementation.
+Agent mode uses ReferenceAssembler (built into core/prompt/renderer.py).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+
+from core.interfaces import PromptAssembler
+from core.paths import resolve_file_list, resolve_skeleton, resolve_skeleton_summary
+
+logger = logging.getLogger(__name__)
+
+# Configurable limits (loaded from config in production)
+DEFAULT_MAX_SOURCE_BYTES = 300_000
+DEFAULT_MAX_SKELETON_BYTES = 50_000
+
+
+class InlinePromptAssembler(PromptAssembler):
+    """Inlines source code and skeleton content directly into prompts.
+
+    Used in CLI mode where the LLM cannot read files itself.
+    """
+
+    def __init__(self, max_source_bytes: int = DEFAULT_MAX_SOURCE_BYTES,
+                 max_skeleton_bytes: int = DEFAULT_MAX_SKELETON_BYTES,
+                 preset: dict | None = None):
+        limits = preset.get("limits", {}) if preset else {}
+        self._max_source = limits.get("max_source_inline_bytes", max_source_bytes)
+        self._max_skeleton = limits.get("max_skeleton_inline_bytes", max_skeleton_bytes)
+        self._global_view_types = self._load_global_view_types(preset)
+
+    @staticmethod
+    def _load_global_view_types(preset: dict | None) -> set[str]:
+        """Load global_view doc types from preset config dynamically."""
+        if not preset:
+            return {"source-tree-analysis", "index"}
+        from core.preset import get_global_view_types
+        return set(get_global_view_types(preset))
+
+    def resolve_file_list(self, module_dir: Path, doc_type: str,
+                         file_list_override: str | None = None) -> str:
+        """Return full file list content (inlined).
+
+        For global_view doc types, returns skeleton content instead of file paths.
+        """
+        # Global_View_Doc: LLM needs skeleton content, not file path list
+        if doc_type in self._global_view_types:
+            return self._inline_skeleton_for_global_view(module_dir)
+
+        # Shard override: read the override file directly
+        if file_list_override:
+            override_path = Path(file_list_override)
+            if override_path.exists():
+                content = override_path.read_text(encoding="utf-8").strip()
+                return content if content else "(no matching files)"
+
+        fl = resolve_file_list(module_dir, doc_type)
+        if fl is None:
+            return "(file list does not exist)"
+        content = fl.read_text(encoding="utf-8").strip()
+        return content if content else "(no matching files)"
+
+    def _inline_skeleton_for_global_view(self, module_dir: Path) -> str:
+        """Inline skeleton summary content for global_view doc types in CLI mode."""
+        summary = resolve_skeleton_summary(module_dir)
+        if summary and summary.exists():
+            try:
+                raw = summary.read_text(encoding="utf-8")
+                size_kb = len(raw.encode("utf-8")) / 1024
+                if size_kb <= self._max_skeleton / 1024:
+                    return f"Below is the skeleton summary content ({size_kb:.1f}KB), generate the document based on this:\n\n```json\n{raw}\n```"
+                # Truncate if too large
+                truncated = raw.encode("utf-8")[:self._max_skeleton].decode("utf-8", errors="ignore")
+                return f"Below is the skeleton summary content (truncated to {self._max_skeleton//1024}KB):\n\n```json\n{truncated}\n```\n[skeleton truncated]"
+            except (OSError, UnicodeDecodeError):
+                pass
+        # Fallback to full skeleton
+        resolved = resolve_skeleton(module_dir)
+        if resolved and resolved.exists():
+            try:
+                raw = resolved.read_text(encoding="utf-8")
+                if len(raw.encode("utf-8")) <= self._max_skeleton:
+                    return f"Below is the skeleton content:\n\n```json\n{raw}\n```"
+            except (OSError, UnicodeDecodeError):
+                pass
+        return "(skeleton does not exist, cannot generate global view document)"
+
+    def resolve_source_content(self, module_dir: Path, doc_type: str, source_cache: Path) -> str:
+        """Read and inline source files with priority ordering.
+
+        For global_view doc types, returns empty (skeleton is the input, not source).
+        """
+        # Global_View_Doc: skeleton is the input, not source files
+        if doc_type in self._global_view_types:
+            return ""
+        fl = resolve_file_list(module_dir, doc_type)
+        if fl is None or not fl.exists():
+            return ""
+
+        file_paths = [l.strip() for l in fl.read_text(encoding="utf-8").splitlines()
+                      if l.strip() and not l.strip().startswith("#")]
+        if not file_paths:
+            return ""
+
+        return self._inline_files(file_paths, source_cache, module_dir)
+
+    def resolve_source_content_from_paths(
+        self, module_dir: Path, doc_type: str, source_cache: Path, file_paths: list[str]
+    ) -> str:
+        """Read and inline source files from an explicit file path list (shard override)."""
+        if doc_type in self._global_view_types:
+            return ""
+        cleaned = [l.strip() for l in file_paths if l.strip() and not l.strip().startswith("#")]
+        if not cleaned:
+            return ""
+        return self._inline_files(cleaned, source_cache, module_dir)
+
+    def _inline_files(self, file_paths: list[str], source_cache: Path, module_dir: Path) -> str:
+        """Shared logic to inline source files with byte limit."""
+
+        # Build priority map from skeleton
+        priority = _build_priority_map(module_dir)
+        source_str = str(source_cache.resolve())
+
+        def sort_key(fpath: str):
+            rel = fpath
+            if source_str in fpath:
+                rel = fpath[len(source_str):].lstrip("/\\")
+            info = priority.get(rel, {})
+            return (-info.get("high_methods", 0), -info.get("total_lines", 0), fpath)
+
+        file_paths.sort(key=sort_key)
+
+        # Read and concatenate with byte limit
+        parts: list[str] = []
+        total_bytes = 0
+        max_single = self._max_source // 6
+
+        for fpath in file_paths:
+            src = source_cache / fpath if not Path(fpath).is_absolute() else Path(fpath)
+            if not src.exists():
+                continue
+            try:
+                content = src.read_text(encoding="utf-8")
+            except (OSError, UnicodeDecodeError):
+                continue
+
+            if len(content.encode("utf-8")) > max_single:
+                lines = content.splitlines(keepends=True)
+                truncated = []
+                acc = 0
+                for line in lines:
+                    acc += len(line.encode("utf-8"))
+                    if acc > max_single:
+                        break
+                    truncated.append(line)
+                content = "".join(truncated) + f"\n// [truncated: {len(truncated)}/{len(lines)} lines]\n"
+
+            ext = Path(fpath).suffix.lstrip(".")
+            block = f"### {Path(fpath).name}\n```{ext or 'text'}\n{content}\n```\n"
+            block_bytes = len(block.encode("utf-8"))
+
+            if total_bytes + block_bytes > self._max_source:
+                remaining = len(file_paths) - len(parts)
+                parts.append(f"\n[truncated — {remaining} files omitted]\n")
+                break
+
+            parts.append(block)
+            total_bytes += block_bytes
+
+        return "\n".join(parts)
+
+    def resolve_skeleton_content(self, module_dir: Path) -> str:
+        """Read skeleton content (summary preferred for large skeletons)."""
+        resolved = resolve_skeleton(module_dir)
+        if resolved is None:
+            return "(skeleton does not exist)"
+
+        # Try full skeleton first
+        if resolved.is_file():
+            try:
+                raw = resolved.read_text(encoding="utf-8")
+                if len(raw.encode("utf-8")) <= self._max_skeleton:
+                    return f"```json\n{raw}\n```"
+            except (OSError, UnicodeDecodeError):
+                pass
+
+        # Fallback to summary
+        summary = resolve_skeleton_summary(module_dir)
+        if summary:
+            try:
+                raw = summary.read_text(encoding="utf-8")
+                if len(raw.encode("utf-8")) <= self._max_skeleton:
+                    return f"```json\n{raw}\n```"
+                truncated = raw.encode("utf-8")[:self._max_skeleton].decode("utf-8", errors="ignore")
+                return f"```json\n{truncated}\n```\n[skeleton truncated]"
+            except (OSError, UnicodeDecodeError):
+                pass
+
+        return "(skeleton too large and no summary available)"
+
+    def should_append_source(self) -> bool:
+        """CLI mode appends source as appendix if not already in template."""
+        return True
+
+
+def _build_priority_map(module_dir: Path) -> dict[str, dict]:
+    """Build file priority map from skeleton summary."""
+    summary = resolve_skeleton_summary(module_dir)
+    if not summary:
+        return {}
+    try:
+        entries = json.loads(summary.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return {}
+
+    priority: dict[str, dict] = {}
+    for entry in entries:
+        fpath = entry.get("file", "")
+        if fpath:
+            high = len(entry.get("high_complexity_methods", []))
+            priority[fpath] = {"high_methods": high, "total_lines": entry.get("total_lines", 0)}
+    return priority

engine/confirm.py

@@ -0,0 +1,65 @@
+"""Interactive confirmation — dispatch plan review and rule approval.
+
+Provides terminal-based confirmation prompts for CLI pipeline gates.
+Supports --no-confirm bypass for CI/automation.
+"""
+
+from __future__ import annotations
+
+import sys
+
+
+def confirm_dispatch_plan(plan_text: str, auto_confirm: bool = False) -> bool:
+    """Display dispatch plan and wait for user confirmation.
+
+    Args:
+        plan_text: Formatted dispatch plan to display
+        auto_confirm: If True, skip confirmation (--no-confirm mode)
+
+    Returns:
+        True if confirmed, False if rejected
+    """
+    if auto_confirm:
+        return True
+
+    print("\n" + "=" * 70)
+    print("  Dispatch Preview (Gate 3A)")
+    print("=" * 70)
+    print(plan_text)
+    print("=" * 70)
+    print()
+
+    try:
+        answer = input("Confirm execution? (y/yes to continue, anything else to abort): ").strip().lower()
+    except (EOFError, KeyboardInterrupt):
+        return False
+
+    return answer in ("y", "yes")
+
+
+def confirm_classification_rules(rules_text: str, auto_confirm: bool = False) -> bool:
+    """Display proposed classification rules and wait for confirmation.
+
+    Args:
+        rules_text: Formatted rules to display
+        auto_confirm: If True, skip confirmation
+
+    Returns:
+        True if user approves writing rules to custom_rules.yaml
+    """
+    if auto_confirm:
+        return True
+
+    print("\n" + "-" * 50)
+    print("  LLM Classification Rule Suggestions")
+    print("-" * 50)
+    print(rules_text)
+    print("-" * 50)
+    print()
+
+    try:
+        answer = input("Write to custom_rules.yaml? (y/yes to confirm, n/no for this run only): ").strip().lower()
+    except (EOFError, KeyboardInterrupt):
+        return False
+
+    return answer in ("y", "yes")