source-kb 0.2.2__py3-none-any.whl

core/skeleton/loader.py

@@ -0,0 +1,162 @@
+"""Skeleton loading module — unified entry point for loading skeleton entries + metadata.
+
+Wraps core.skeleton.file_list.load_skeleton() and adds metadata loading
+from .meta/parser-info.json.
+
+Usage:
+    from core.skeleton.loader import load_skeleton_entries
+
+    entries, metadata = load_skeleton_entries(module_dir)
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from core.paths import meta_dir, resolve_skeleton
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def load_skeleton_entries(
+    module_dir: Path,
+    *,
+    max_shards: int = 50,
+) -> tuple[list[dict[str, Any]], dict[str, Any]]:
+    """Load skeleton entries and associated metadata from a module directory.
+
+    Handles:
+    - New paths (.meta/skeleton/) and legacy paths (.skeleton.json / .skeleton/)
+    - Single file vs shards directory
+    - dict format (data/files/classes key) vs list format
+    - Metadata loading from .meta/parser-info.json
+
+    Args:
+        module_dir: Module knowledge base directory.
+        max_shards: Maximum number of shard files to read from a shards directory.
+
+    Returns:
+        Tuple of (entries, metadata):
+        - entries: Flat list of skeleton file entry dicts.
+        - metadata: Dict with parser info, timestamp, entry count, total size, etc.
+    """
+    resolved = resolve_skeleton(module_dir)
+    if resolved is None:
+        return [], _empty_metadata()
+
+    if resolved.is_dir():
+        entries, total_size = _load_from_dir(resolved, max_shards)
+    else:
+        entries, total_size = _load_from_file(resolved)
+
+    # Load parser metadata
+    metadata = _load_parser_metadata(module_dir)
+    metadata["entry_count"] = len(entries)
+    metadata["total_size_bytes"] = total_size
+    metadata["source_path"] = str(resolved)
+
+    return entries, metadata
+
+
+# ---------------------------------------------------------------------------
+# Internal: skeleton loading
+# ---------------------------------------------------------------------------
+
+
+def _load_from_file(path: Path) -> tuple[list[dict], int]:
+    """Load skeleton from a single JSON file."""
+    try:
+        raw = path.read_text(encoding="utf-8")
+    except (OSError, UnicodeDecodeError) as e:
+        logger.warning("Failed to read skeleton file %s: %s", path, e)
+        return [], 0
+
+    total_size = len(raw.encode("utf-8"))
+    entries = _parse_skeleton_data(raw)
+    return entries, total_size
+
+
+def _load_from_dir(dir_path: Path, max_shards: int) -> tuple[list[dict], int]:
+    """Load skeleton from a shards directory."""
+    entries: list[dict] = []
+    total_size = 0
+
+    for f in sorted(dir_path.glob("*.json"))[:max_shards]:
+        try:
+            raw = f.read_text(encoding="utf-8")
+        except (OSError, UnicodeDecodeError):
+            continue
+        total_size += len(raw.encode("utf-8"))
+        entries.extend(_parse_skeleton_data(raw))
+
+    return entries, total_size
+
+
+def _parse_skeleton_data(raw: str) -> list[dict]:
+    """Parse skeleton JSON string, handling both dict and list formats."""
+    try:
+        data = json.loads(raw)
+    except (json.JSONDecodeError, ValueError):
+        return []
+
+    if isinstance(data, list):
+        return data
+    if isinstance(data, dict):
+        for key in ("data", "files", "classes"):
+            if key in data:
+                val = data[key]
+                if isinstance(val, list):
+                    return val
+        return []
+    return []
+
+
+# ---------------------------------------------------------------------------
+# Internal: metadata loading
+# ---------------------------------------------------------------------------
+
+
+def _load_parser_metadata(module_dir: Path) -> dict[str, Any]:
+    """Load parser metadata from .meta/parser-info.json if available."""
+    parser_info_path = meta_dir(module_dir) / "parser-info.json"
+    if not parser_info_path.exists():
+        return _empty_metadata()
+
+    try:
+        data = json.loads(parser_info_path.read_text(encoding="utf-8"))
+        if not isinstance(data, dict):
+            return _empty_metadata()
+        return {
+            "parser_name": data.get("parser_name", "unknown"),
+            "parser_version": data.get("parser_version", ""),
+            "timestamp": data.get("timestamp", ""),
+            "parse_duration_ms": data.get("parse_duration_ms", 0),
+            "entry_count": data.get("entry_count", 0),
+            "total_size_bytes": data.get("total_size_bytes", 0),
+            "source_path": "",
+        }
+    except (json.JSONDecodeError, OSError) as e:
+        logger.warning("Failed to read parser-info.json: %s", e)
+        return _empty_metadata()
+
+
+def _empty_metadata() -> dict[str, Any]:
+    """Return empty metadata dict with default values."""
+    return {
+        "parser_name": "unknown",
+        "parser_version": "",
+        "timestamp": "",
+        "parse_duration_ms": 0,
+        "entry_count": 0,
+        "total_size_bytes": 0,
+        "source_path": "",
+    }

core/skeleton/merge.py

@@ -0,0 +1,278 @@
+"""Shard merge and post-merge refinement.
+
+Fast (deterministic) merge of split document shards into single files.
+Post-merge refinement: duplicate removal, term consistency, anchor addition.
+
+LLM-based smart merge lives in cli/dedup.py (requires LlmStrategy).
+
+Usage:
+    from core.skeleton.merge import find_shards, merge_shards, refine_merged_doc
+
+    shards = find_shards(module_dir, "business-logic")
+    content = merge_shards(shards)
+    result = refine_merged_doc(merged_path)
+    if result.changed:
+        result.apply()
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+# ---------------------------------------------------------------------------
+# Shard discovery and merge
+# ---------------------------------------------------------------------------
+
+
+def find_shards(module_dir: Path, prefix: str) -> list[Path]:
+    """Find and sort all shard files for a given doc type prefix."""
+    shards = sorted(module_dir.glob(f"{prefix}-*.md"))
+    if not shards:
+        shards = sorted(module_dir.glob(f".shard-{prefix}-*.md"))
+    return shards
+
+
+def merge_shards(shards: list[Path], order: list[str] | None = None) -> str:
+    """Fast merge: concatenate shards, keeping first title only.
+
+    Args:
+        shards: Ordered list of shard file paths
+        order: Optional explicit ordering by shard suffix names
+
+    Returns:
+        Merged markdown content
+    """
+    if not shards:
+        return ""
+
+    if order:
+        shards = _reorder_shards(shards, order)
+
+    first = shards[0].read_text(encoding="utf-8")
+    lines = first.split("\n", 1)
+    title = lines[0] + "\n\n"
+
+    parts: list[str] = []
+    for shard in shards:
+        content = shard.read_text(encoding="utf-8")
+        # Remove first-level heading (keep only one)
+        content = re.sub(r'^#[^#].*\r?\n?', '', content, count=1)
+        parts.append(content.strip())
+
+    return title + "\n\n".join(parts) + "\n"
+
+
+def _reorder_shards(shards: list[Path], order: list[str]) -> list[Path]:
+    """Reorder shards by explicit suffix list."""
+    ordered: list[Path] = []
+    remaining = list(shards)
+    for suffix in order:
+        for s in remaining:
+            if suffix in s.stem:
+                ordered.append(s)
+                remaining.remove(s)
+                break
+    ordered.extend(remaining)
+    return ordered
+
+
+# ---------------------------------------------------------------------------
+# Post-merge refinement (rule-based, no LLM)
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class RefineResult:
+    """Result of post-merge refinement."""
+    original_path: Path
+    original_content: str
+    refined_content: str
+    issues: list[str] = field(default_factory=list)
+    duplicates_removed: int = 0
+    terms_fixed: int = 0
+    anchors_added: int = 0
+
+    @property
+    def changed(self) -> bool:
+        return self.refined_content != self.original_content
+
+    def apply(self) -> None:
+        """Write refined content back to file."""
+        if self.changed:
+            self.original_path.write_text(self.refined_content, encoding="utf-8")
+
+    def summary(self) -> str:
+        parts = []
+        if self.duplicates_removed:
+            parts.append(f"dedup {self.duplicates_removed}")
+        if self.terms_fixed:
+            parts.append(f"terms {self.terms_fixed}")
+        if self.anchors_added:
+            parts.append(f"anchors {self.anchors_added}")
+        return ", ".join(parts) if parts else "no changes"
+
+
+def refine_merged_doc(
+    merged_path: Path,
+    term_aliases: dict[str, list[str]] | None = None,
+) -> RefineResult:
+    """Rule-based refinement: dedup paragraphs, fix terms, add anchors.
+
+    Args:
+        merged_path: Path to the merged document
+        term_aliases: {standard_term: [alias1, alias2]} for consistency fixes
+
+    Returns:
+        RefineResult with refined content and issue summary
+    """
+    try:
+        content = merged_path.read_text(encoding="utf-8")
+    except (OSError, UnicodeDecodeError):
+        return RefineResult(original_path=merged_path, original_content="", refined_content="",
+                            issues=["file read failed"])
+
+    result = RefineResult(original_path=merged_path, original_content=content, refined_content=content)
+
+    # Step 1: Remove duplicate paragraphs
+    result.refined_content, dups = _remove_duplicates(result.refined_content)
+    result.duplicates_removed = dups
+    if dups:
+        result.issues.append(f"{dups} duplicate paragraphs removed")
+
+    # Step 2: Term consistency
+    if term_aliases:
+        result.refined_content, fixes = _fix_terms(result.refined_content, term_aliases)
+        result.terms_fixed = fixes
+        if fixes:
+            result.issues.append(f"{fixes} term inconsistencies fixed")
+
+    # Step 3: Add missing anchors
+    result.refined_content, anchors = _add_anchors(result.refined_content)
+    result.anchors_added = anchors
+    if anchors:
+        result.issues.append(f"{anchors} anchors added")
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _normalize(text: str) -> str:
+    """Normalize paragraph for comparison."""
+    text = re.sub(r'\s+', ' ', text.strip())
+    text = re.sub(r"[，。、；：\u201c\u201d\u2018\u2019（）\[\]【】]", "", text)
+    return text.lower()
+
+
+def _remove_duplicates(content: str) -> tuple[str, int]:
+    """Remove duplicate paragraphs (> 80 chars, keep first occurrence)."""
+    paragraphs = content.split("\n\n")
+    seen: set[str] = set()
+    result: list[str] = []
+    removed = 0
+
+    for para in paragraphs:
+        normalized = _normalize(para)
+        if len(normalized) < 80 or para.strip().startswith("#"):
+            result.append(para)
+            continue
+        if normalized in seen:
+            removed += 1
+            continue
+        seen.add(normalized)
+        result.append(para)
+
+    return "\n\n".join(result), removed
+
+
+def _fix_terms(content: str, aliases: dict[str, list[str]]) -> tuple[str, int]:
+    """Replace term aliases with standard terms."""
+    fixes = 0
+    for standard, alias_list in aliases.items():
+        for alias in alias_list:
+            if alias == standard:
+                continue
+            pattern = re.compile(r'(?<![a-zA-Z0-9_])' + re.escape(alias) + r'(?![a-zA-Z0-9_])')
+            content, n = pattern.subn(standard, content)
+            fixes += n
+    return content, fixes
+
+
+def _add_anchors(content: str) -> tuple[str, int]:
+    """Add HTML anchors to ## headings that don't have one."""
+    lines = content.splitlines()
+    added = 0
+    result: list[str] = []
+
+    for i, line in enumerate(lines):
+        if line.startswith("## ") and not line.startswith("### "):
+            title = line.lstrip("#").strip()
+            slug = re.sub(r'[^\w\u4e00-\u9fff-]', '-', title).strip('-').lower()
+            has_anchor = result and '<a id=' in result[-1]
+            if not has_anchor and slug:
+                result.append(f'<a id="{slug}"></a>')
+                added += 1
+        result.append(line)
+
+    return "\n".join(result), added
+
+
+# ---------------------------------------------------------------------------
+# CLI entry point
+# ---------------------------------------------------------------------------
+
+
+def _cli_main() -> None:
+    """CLI: merge shards in a module directory.
+
+    Usage:
+        python -X utf8 core/skeleton/merge.py --dir <module_dir>
+        python -X utf8 core/skeleton/merge.py --prefix business-logic --order shard1,shard2 --dir <module_dir>
+    """
+    import argparse
+    import sys
+
+    parser = argparse.ArgumentParser(description="Merge shard documents in a module directory")
+    parser.add_argument("--dir", required=True, help="Module knowledge directory containing shard files")
+    parser.add_argument("--prefix", help="Only merge shards with this doc-type prefix (e.g. business-logic)")
+    parser.add_argument("--order", help="Comma-separated shard suffix ordering")
+    args = parser.parse_args()
+
+    module_dir = Path(args.dir)
+    if not module_dir.is_dir():
+        print(f"Error: directory not found: {module_dir}", file=sys.stderr)
+        sys.exit(1)
+
+    order = args.order.split(",") if args.order else None
+
+    if args.prefix:
+        prefixes = [args.prefix]
+    else:
+        if order:
+            print("Warning: --order ignored without --prefix (order is prefix-specific)", file=sys.stderr)
+            order = None
+        shard_files = sorted(module_dir.glob("*-shard-*.md"))
+        prefixes = sorted({f.stem.rsplit("-shard-", 1)[0] for f in shard_files})
+
+    if not prefixes:
+        print("No shard files found.", file=sys.stderr)
+        sys.exit(0)
+
+    for prefix in prefixes:
+        shards = find_shards(module_dir, prefix)
+        if not shards:
+            continue
+        merged = merge_shards(shards, order=order)
+        out_path = module_dir / f"{prefix}.md"
+        out_path.write_text(merged, encoding="utf-8")
+        print(f"Merged {len(shards)} shards \u2192 {out_path.name} ({len(merged)} chars)")
+
+
+if __name__ == "__main__":
+    _cli_main()

core/skeleton/merge_delta.py

@@ -0,0 +1,229 @@
+"""Skeleton delta merge — merges incremental skeleton updates into existing skeleton.
+
+Handles both single-file (.skeleton.json) and sharded (.skeleton/ directory) formats.
+Pure logic, no CLI I/O, no LLM calls.
+
+Usage:
+    from core.skeleton.merge_delta import merge_delta, merge_entries
+
+    stats = merge_delta(delta_path, module_dir)
+    print(f"Replaced: {stats.replaced}, Added: {stats.added}")
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Data classes
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class MergeStats:
+    """Statistics from a delta merge operation."""
+
+    replaced: int = 0
+    added: int = 0
+    unmatched: list[str] | None = None
+
+    @property
+    def total_changes(self) -> int:
+        return self.replaced + self.added
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def merge_delta(
+    delta_path: Path,
+    module_dir: Path,
+    dry_run: bool = False,
+    cleanup: bool = True,
+) -> MergeStats:
+    """Merge a skeleton delta file into the existing skeleton.
+
+    Auto-detects target format (single file vs sharded directory).
+
+    Args:
+        delta_path: Path to .skeleton-delta.json
+        module_dir: Module knowledge directory
+        dry_run: If True, compute stats without writing
+        cleanup: If True, delete delta file after successful merge
+
+    Returns:
+        MergeStats with replaced/added counts
+    """
+    from core.paths import resolve_skeleton
+
+    if not delta_path.exists():
+        logger.warning("Delta file not found: %s", delta_path)
+        return MergeStats()
+
+    delta_entries = load_json(delta_path)
+    if not delta_entries:
+        return MergeStats()
+
+    skel = resolve_skeleton(module_dir)
+    if skel is None:
+        logger.warning("No skeleton found in %s", module_dir)
+        return MergeStats()
+
+    if skel.is_dir():
+        stats = _merge_to_shards(skel, delta_entries, dry_run)
+    else:
+        stats = _merge_to_single(skel, delta_entries, dry_run)
+
+    if not dry_run and cleanup and delta_path.exists():
+        delta_path.unlink()
+
+    return stats
+
+
+def merge_entries(
+    existing: list[dict],
+    delta: list[dict],
+) -> tuple[list[dict], int, int]:
+    """Merge delta entries into existing entries by file path.
+
+    Args:
+        existing: Current skeleton entries
+        delta: New/updated entries to merge
+
+    Returns:
+        (merged_entries, replaced_count, added_count)
+    """
+    index = {e["file"]: i for i, e in enumerate(existing) if "file" in e}
+    replaced = 0
+    added = 0
+
+    for entry in delta:
+        fp = entry.get("file", "")
+        if not fp:
+            continue
+        if fp in index:
+            existing[index[fp]] = entry
+            replaced += 1
+        else:
+            existing.append(entry)
+            index[fp] = len(existing) - 1
+            added += 1
+
+    return existing, replaced, added
+
+
+def infer_shard_name(filepath: str, shard_names: list[str]) -> str | None:
+    """Infer which shard a file belongs to based on package path.
+
+    Shard filenames: com_example_app_service.json (package with underscores)
+    File paths: com/example/app/service/SomeClass.java
+
+    Matches the longest shard prefix.
+    """
+    parts = filepath.replace("\\", "/").rsplit("/", 1)
+    if len(parts) < 2:
+        return None
+    dir_path = parts[0]
+    dir_key = dir_path.replace("/", "_")
+
+    best_match = None
+    best_length = 0
+
+    for shard in shard_names:
+        shard_key = shard.replace(".json", "")
+        if dir_key.startswith(shard_key) and len(shard_key) > best_length:
+            best_match = shard
+            best_length = len(shard_key)
+
+    return best_match
+
+
+# ---------------------------------------------------------------------------
+# File I/O helpers
+# ---------------------------------------------------------------------------
+
+
+def load_json(path: Path) -> list[dict]:
+    """Load skeleton JSON, normalize to entry list."""
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError) as e:
+        logger.warning("Failed to load JSON %s: %s", path, e)
+        return []
+
+    if isinstance(data, list):
+        return data
+    return data.get("data", data.get("files", []))
+
+
+def save_json(path: Path, entries: list[dict]) -> None:
+    """Save skeleton entries as JSON."""
+    path.write_text(
+        json.dumps(entries, ensure_ascii=False, indent=1),
+        encoding="utf-8",
+    )
+
+
+# ---------------------------------------------------------------------------
+# Internal merge implementations
+# ---------------------------------------------------------------------------
+
+
+def _merge_to_single(skel_path: Path, delta_entries: list[dict], dry_run: bool) -> MergeStats:
+    """Merge delta into a single skeleton file."""
+    existing = load_json(skel_path)
+    merged, replaced, added = merge_entries(existing, delta_entries)
+
+    if not dry_run:
+        save_json(skel_path, merged)
+
+    return MergeStats(replaced=replaced, added=added)
+
+
+def _merge_to_shards(shard_dir: Path, delta_entries: list[dict], dry_run: bool) -> MergeStats:
+    """Merge delta into sharded skeleton directory."""
+    shard_files = sorted(shard_dir.glob("*.json"))
+    shard_names = [f.name for f in shard_files]
+
+    if not shard_names:
+        logger.warning("Skeleton shard directory is empty: %s", shard_dir)
+        return MergeStats()
+
+    # Group delta entries by target shard
+    shard_groups: dict[str, list[dict]] = {}
+    unmatched: list[str] = []
+
+    for entry in delta_entries:
+        fp = entry.get("file", "")
+        shard = infer_shard_name(fp, shard_names)
+        if shard:
+            shard_groups.setdefault(shard, []).append(entry)
+        else:
+            unmatched.append(fp)
+
+    total_replaced = 0
+    total_added = 0
+
+    for shard_name, entries in shard_groups.items():
+        shard_path = shard_dir / shard_name
+        existing = load_json(shard_path)
+        merged, replaced, added = merge_entries(existing, entries)
+        total_replaced += replaced
+        total_added += added
+
+        if not dry_run:
+            save_json(shard_path, merged)
+
+    return MergeStats(
+        replaced=total_replaced,
+        added=total_added,
+        unmatched=unmatched if unmatched else None,
+    )