source-kb 0.2.2__py3-none-any.whl

core/skeleton/diff_doc.py

@@ -0,0 +1,248 @@
+"""Document-skeleton structural comparison tool.
+
+Compares document content against skeleton entries to identify
+inconsistencies (missing or stale items).
+
+Usage:
+    python -m core.skeleton diff-doc --doc-path business-logic.md --skeleton-path .skeleton.json
+
+    from core.skeleton.diff_doc import diff_doc
+    result = diff_doc(doc_path, skeleton_path, doc_type="business-logic")
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Literal
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Data classes
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class DiffFinding:
+    """Single diff finding."""
+
+    item: str  # class/method/field name
+    category: Literal["missing", "stale"]
+    doc_location: str = ""  # section heading where found/expected
+    skeleton_location: str = ""  # file path in skeleton
+    detail: str = ""
+
+
+@dataclass
+class DiffResult:
+    """Complete diff comparison result."""
+
+    doc_type: str
+    findings: list[DiffFinding] = field(default_factory=list)
+    doc_items_count: int = 0
+    skeleton_items_count: int = 0
+
+    @property
+    def missing_count(self) -> int:
+        return sum(1 for f in self.findings if f.category == "missing")
+
+    @property
+    def stale_count(self) -> int:
+        return sum(1 for f in self.findings if f.category == "stale")
+
+    def to_json(self) -> list[dict]:
+        """Convert to audit-compatible JSON format."""
+        return [
+            {
+                "dimension": f"diff_{f.category}",
+                "status": "fail",
+                "detail": f"{f.category}: {f.item} — {f.detail}",
+                "fix": "",
+            }
+            for f in self.findings
+        ]
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def diff_doc(
+    doc_path: Path,
+    skeleton_path: Path,
+    *,
+    doc_type: str | None = None,
+) -> DiffResult:
+    """Compare document against skeleton.
+
+    Comparison rules by doc_type:
+    - data-models: field-level (entity class names)
+    - api-contracts: endpoint-level (controller method names + URLs)
+    - business-logic: method-level (service method names)
+    - default: class-level (class names from skeleton files)
+
+    Args:
+        doc_path: Path to the markdown document
+        skeleton_path: Path to skeleton JSON file or directory
+        doc_type: Document type for type-specific comparison rules
+
+    Returns:
+        DiffResult with findings in audit-compatible format
+    """
+    result = DiffResult(doc_type=doc_type or "unknown")
+
+    # Load skeleton entries
+    skeleton_entries = _load_skeleton(skeleton_path)
+    if not skeleton_entries:
+        return result
+
+    # Load document content
+    try:
+        doc_content = doc_path.read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return result
+
+    # Extract items from skeleton based on doc_type
+    skeleton_items = _extract_skeleton_items(skeleton_entries, doc_type)
+    result.skeleton_items_count = len(skeleton_items)
+
+    # Extract items mentioned in document
+    doc_items = _extract_doc_items(doc_content, doc_type)
+    result.doc_items_count = len(doc_items)
+
+    # Compare: skeleton items not in doc → "missing"
+    doc_items_lower = {item.lower() for item in doc_items}
+    for item, file_path in skeleton_items.items():
+        if item.lower() not in doc_items_lower:
+            result.findings.append(DiffFinding(
+                item=item,
+                category="missing",
+                skeleton_location=file_path,
+                detail=f"In skeleton ({file_path}) but not found in document",
+            ))
+
+    # Compare: doc items not in skeleton → "stale"
+    skeleton_items_lower = {item.lower() for item in skeleton_items}
+    for item in doc_items:
+        if item.lower() not in skeleton_items_lower:
+            result.findings.append(DiffFinding(
+                item=item,
+                category="stale",
+                doc_location="document",
+                detail=f"In document but not found in skeleton (possibly deleted/renamed)",
+            ))
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _load_skeleton(skeleton_path: Path) -> list[dict]:
+    """Load skeleton entries from file or directory."""
+    if skeleton_path.is_dir():
+        entries = []
+        for f in sorted(skeleton_path.glob("*.json")):
+            try:
+                data = json.loads(f.read_text(encoding="utf-8"))
+                if isinstance(data, list):
+                    entries.extend(data)
+            except (json.JSONDecodeError, OSError):
+                continue
+        return entries
+    elif skeleton_path.exists():
+        try:
+            data = json.loads(skeleton_path.read_text(encoding="utf-8"))
+            if isinstance(data, list):
+                return data
+            return data.get("data", data.get("files", []))
+        except (json.JSONDecodeError, OSError):
+            return []
+    return []
+
+
+def _extract_skeleton_items(entries: list[dict], doc_type: str | None) -> dict[str, str]:
+    """Extract relevant items from skeleton based on doc_type.
+
+    Returns: {item_name: file_path}
+    """
+    items: dict[str, str] = {}
+
+    for entry in entries:
+        file_path = entry.get("file", "")
+        classes = entry.get("classes", [])
+        methods = entry.get("methods", [])
+
+        if doc_type in ("data-models",):
+            # Extract class names (entities, DTOs)
+            for cls in classes:
+                name = cls.get("name", "")
+                if name:
+                    items[name] = file_path
+        elif doc_type in ("api-contracts",):
+            # Extract method names from controllers
+            for m in methods:
+                name = m.get("name", "")
+                annotations = m.get("annotations", [])
+                # Look for mapping annotations
+                for a in annotations:
+                    a_name = a if isinstance(a, str) else a.get("name", "")
+                    if "Mapping" in a_name:
+                        items[name] = file_path
+                        break
+        elif doc_type in ("business-logic",):
+            # Extract service method names (high complexity preferred)
+            for m in methods:
+                name = m.get("name", "")
+                complexity = m.get("complexity", "")
+                if complexity == "high" and name:
+                    items[name] = file_path
+        else:
+            # Default: class names
+            for cls in classes:
+                name = cls.get("name", "")
+                if name:
+                    items[name] = file_path
+
+    return items
+
+
+def _extract_doc_items(content: str, doc_type: str | None) -> list[str]:
+    """Extract item names mentioned in the document."""
+    items: list[str] = []
+
+    # Extract from headings (## and ###)
+    for line in content.splitlines():
+        if line.startswith("## ") or line.startswith("### "):
+            heading_text = line.lstrip("#").strip()
+            # Extract class/method names from headings
+            # Common patterns: "## ClassName", "## ClassName (description)", "### methodName"
+            # Extract the first word/identifier
+            match = re.match(r'[`]?(\w+)[`]?', heading_text)
+            if match:
+                items.append(match.group(1))
+
+    # Extract from code blocks and inline code
+    for match in re.finditer(r'`(\w+(?:\.\w+)?)`', content):
+        word = match.group(1)
+        # Filter out common non-identifiers
+        if len(word) > 2 and word[0].isupper():
+            items.append(word)
+
+    # Deduplicate while preserving order
+    seen: set[str] = set()
+    unique: list[str] = []
+    for item in items:
+        if item not in seen:
+            seen.add(item)
+            unique.append(item)
+
+    return unique

core/skeleton/dispatch.py

@@ -0,0 +1,273 @@
+"""Dispatch plan computation — module->doc_type mapping, priority, split triggering.
+
+Computes the full dispatch plan for document generation: which doc_types to generate,
+how many shards each needs, and in what batch order.
+
+No CLI I/O, no LLM calls. Pure computation from config + file system state.
+
+Usage:
+    from core.skeleton.dispatch import compute_dispatch_plan, DispatchPlan
+
+    plan = compute_dispatch_plan(
+        preset=preset, module_dir=module_dir, source_cache=source_cache,
+        mode="readwrite", module_type="service",
+    )
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from core.skeleton.dispatch_source import (
+    get_file_list_with_stats, count_source_lines, skeleton_total_size_kb,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ShardPlan:
+    """Plan for a single shard within a dispatch entry."""
+    name: str
+    files: list[str] = field(default_factory=list)
+    lines: int = 0
+    bytes: int = 0
+    file_count: int = 0
+
+
+@dataclass
+class DispatchEntry:
+    """Single module dispatch plan entry (one doc_type or shard)."""
+    module_name: str = ""
+    doc_type: str = ""
+    doc_filename: str = ""
+    batch_id: str = ""
+    file_count: int = 0
+    total_lines: int = 0
+    total_bytes: int = 0
+    split_count: int = 1
+    priority: int = 0
+    shards: list[ShardPlan] = field(default_factory=list)
+    strategy: str = "single-agent"
+    note: str = ""
+    pending_grouping: str | None = None
+
+
+@dataclass
+class DispatchPlan:
+    """Complete dispatch plan for a module."""
+    module_name: str = ""
+    entries: list[DispatchEntry] = field(default_factory=list)
+    batch_order: list[tuple[str, list[str]]] = field(default_factory=list)
+    source_file_count: int = 0
+    source_total_lines: int = 0
+    skeleton_size_kb: int = 0
+
+    def total_shards(self) -> int:
+        return sum(max(1, e.split_count) for e in self.entries)
+
+    def total_entries(self) -> int:
+        return len(self.entries)
+
+
+# ---------------------------------------------------------------------------
+# Batch order inference
+# ---------------------------------------------------------------------------
+
+_DEFAULT_BATCH_ORDER: list[tuple[str, list[str]]] = [
+    ("1", ["source-tree-analysis"]),
+    ("2A", ["data-models", "enums-and-constants"]),
+    ("2B", ["api-contracts", "architecture", "caching", "messaging"]),
+    ("3", ["business-logic", "utils"]),
+    ("4", ["index"]),
+]
+
+_LEVELS: dict[int, list[str]] = {
+    1: ["source-tree", "structure", "tree", "enums", "constants", "enum"],
+    2: ["data-models", "data-model", "entities", "entity", "models", "schema",
+        "database-access", "database", "dao", "mapper"],
+    3: ["api", "contract", "architecture", "config", "redis", "kafka", "mq", "cache",
+        "middleware", "scheduled", "error-handling", "security", "aop", "interceptor",
+        "async", "event", "external"],
+    4: ["business", "logic", "service", "workflow"],
+    5: ["utils", "util", "helper", "tool"],
+    6: ["index", "summary", "overview"],
+}
+
+
+def infer_batch_order(preset: dict[str, Any]) -> list[tuple[str, list[str]]]:
+    """Infer batch order from preset doc_types (language-agnostic)."""
+    from core.preset import get_doc_type_mapping
+    doc_mapping = get_doc_type_mapping(preset)
+    if not doc_mapping:
+        return list(_DEFAULT_BATCH_ORDER)
+
+    def _get_level(doc_type: str) -> int:
+        dt_lower = doc_type.lower()
+        for level, keywords in _LEVELS.items():
+            for kw in keywords:
+                if kw in dt_lower:
+                    return level
+        return 4
+
+    level_groups: dict[int, list[str]] = {}
+    for dt in doc_mapping:
+        level_groups.setdefault(_get_level(dt), []).append(dt)
+
+    result = [(str(level), dts) for level, dts in sorted(level_groups.items())]
+    return result if result else list(_DEFAULT_BATCH_ORDER)
+
+
+# ---------------------------------------------------------------------------
+# Optional doc detection
+# ---------------------------------------------------------------------------
+
+_OPTIONAL_DOCS: frozenset[str] = frozenset({
+    "caching", "messaging", "external-integrations", "scheduled-tasks",
+    "error-handling", "security", "async-and-events", "aop-and-interceptors",
+    "database-access", "observability",
+})
+
+
+def detect_optional_doc(source_cache: Path, preset: dict[str, Any], doc_filename: str) -> bool:
+    """Scan source to detect if an optional doc type should be triggered."""
+    if not source_cache.is_dir():
+        return False
+    classification = preset.get("file_classification", {})
+    trigger_categories: set[str] = set()
+    for cat_name, cat_config in classification.items():
+        if isinstance(cat_config, dict) and doc_filename in cat_config.get("affects", []):
+            trigger_categories.add(cat_name)
+    if not trigger_categories:
+        return False
+    from core.preset_classify import classify_file
+    for root, _, filenames in os.walk(source_cache):
+        root_path = Path(root)
+        if "/test/" in str(root_path).replace("\\", "/"):
+            continue
+        for fn in filenames:
+            if not fn.endswith(".java"):
+                continue
+            rel = str((root_path / fn).relative_to(source_cache)).replace("\\", "/")
+            if trigger_categories & set(classify_file(preset, rel)):
+                return True
+    return False
+
+
+# ---------------------------------------------------------------------------
+# Compute dispatch plan
+# ---------------------------------------------------------------------------
+
+_STRATEGY_LABELS = {
+    "community": "community",
+    "llm": "llm",
+    "package": "package",
+    "simple": "simple",
+    "cache": "cache",
+}
+
+
+def compute_dispatch_plan(
+    preset: dict[str, Any], module_dir: Path, source_cache: Path,
+    mode: str = "readwrite", module_name: str = "", module_type: str = "service",
+) -> DispatchPlan:
+    """Compute the full dispatch plan for a module."""
+    from core.skeleton.split import SplitConfig, compute_splits
+    from core.skeleton.split_plan import plan_splits
+    from core.skeleton.loader import load_skeleton_entries
+    from core.preset import get_doc_type_mapping, get_doc_filename
+
+    dt_mapping = get_doc_type_mapping(preset)
+    batch_order = infer_batch_order(preset)
+    split_config = SplitConfig.from_preset(preset, mode=mode)
+
+    module_types_config = preset.get("module_types", {})
+    type_config = module_types_config.get(module_type, {})
+    skip_docs = set(type_config.get("skip", []))
+    skip_all = "*" in skip_docs
+
+    skel_size_kb = skeleton_total_size_kb(module_dir)
+    src_files, src_lines = count_source_lines(source_cache)
+    entries: list[DispatchEntry] = []
+
+    for batch_id, doc_types in batch_order:
+        for dt in doc_types:
+            if dt not in dt_mapping:
+                continue
+            doc_filename = dt_mapping[dt]
+            if skip_all or doc_filename in skip_docs:
+                continue
+            files = get_file_list_with_stats(module_dir, dt, source_cache, dt_mapping)
+            file_count = len(files)
+            if dt in _OPTIONAL_DOCS and file_count == 0:
+                doc_fn = get_doc_filename(preset, dt, strict=False)
+                if not detect_optional_doc(source_cache, preset, doc_fn):
+                    continue
+            total_lines_dt = sum(f["lines"] for f in files)
+            total_bytes_dt = sum(f.get("bytes", 0) for f in files)
+
+            is_global_view = dt in split_config.skip_file_count_doc_types
+            if is_global_view:
+                total_bytes_dt = skel_size_kb * 1024
+                total_lines_dt = 0
+
+            n_splits = compute_splits(
+                split_config, total_bytes=total_bytes_dt, total_lines=total_lines_dt,
+                file_count=file_count, doc_type=dt,
+            )
+            if is_global_view:
+                n_splits = 1
+
+            shard_plans: list[ShardPlan] = []
+            pending_grouping: str | None = None
+            split_strategy_name: str = ""
+            if n_splits > 1 and files:
+                try:
+                    skel_entries, _ = load_skeleton_entries(module_dir)
+                    split_result = plan_splits(
+                        entries=skel_entries, file_list=files,
+                        split_config=split_config, doc_type=dt,
+                        module_dir=module_dir,
+                    )
+                    split_strategy_name = split_result.strategy
+                    if split_result.pending_agent_grouping:
+                        pending_grouping = split_result.grouping_request_path
+                    else:
+                        for s in split_result.splits:
+                            shard_plans.append(ShardPlan(
+                                name=s.get("name", ""),
+                                files=s.get("files", []),
+                                lines=s.get("lines", 0),
+                                bytes=s.get("bytes", 0),
+                                file_count=s.get("file_count", 0),
+                            ))
+                        if shard_plans:
+                            n_splits = len(shard_plans)
+                except Exception as e:
+                    logger.warning("plan_splits failed for %s/%s: %s", module_name, dt, e)
+
+            if n_splits <= 1:
+                strategy_display = "single-agent"
+            else:
+                label = _STRATEGY_LABELS.get(split_strategy_name, split_strategy_name)
+                strategy_display = f"{label} x{n_splits}"
+
+            entries.append(DispatchEntry(
+                module_name=module_name, doc_type=dt, doc_filename=doc_filename,
+                batch_id=batch_id, file_count=file_count, total_lines=total_lines_dt,
+                total_bytes=total_bytes_dt, split_count=n_splits,
+                shards=shard_plans,
+                strategy=strategy_display,
+                note="global_view: skeleton summary only" if is_global_view else "",
+                pending_grouping=pending_grouping,
+            ))
+
+    return DispatchPlan(
+        module_name=module_name, entries=entries, batch_order=batch_order,
+        source_file_count=src_files, source_total_lines=src_lines,
+        skeleton_size_kb=skel_size_kb,
+    )