source-kb 0.2.2__py3-none-any.whl

core/skeleton/split_feedback.py

@@ -0,0 +1,174 @@
+"""Split feedback — record and query split execution results for adaptive tuning.
+
+Records each split execution's quality metrics and provides historical best
+resolution parameter for future splits. Lightweight JSON persistence.
+
+Usage:
+    from core.skeleton.split_feedback import SplitRecord, record_split_result, get_best_resolution
+
+    record_split_result(module_dir, record)
+    best = get_best_resolution(module_dir, "business-logic")
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+FEEDBACK_FILENAME = "split-feedback.json"
+MAX_RECORDS = 20
+
+
+# ---------------------------------------------------------------------------
+# Data classes
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class SplitRecord:
+    """A single split execution record."""
+
+    doc_type: str
+    strategy: str  # "community" | "package" | "simple" | "single"
+    resolution: float = 1.0
+    n_splits: int = 1
+    coverage_score: float = 0.0
+    quality_score: float = 0.0
+    issues_count: int = 0
+    merge_duplicates: int = 0
+    timestamp: str = ""
+
+    def __post_init__(self):
+        if not self.timestamp:
+            self.timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    @property
+    def composite_score(self) -> float:
+        """Compute composite score (0-1).
+
+        Weights: coverage 40%, quality 40%, issue penalty 10%, duplicate penalty 10%.
+        """
+        issue_penalty = min(self.issues_count * 0.02, 0.1)
+        dup_penalty = min(self.merge_duplicates * 0.02, 0.1)
+        score = (
+            self.coverage_score * 0.4
+            + self.quality_score * 0.4
+            + (0.1 - issue_penalty)
+            + (0.1 - dup_penalty)
+        )
+        return round(max(0.0, min(1.0, score)), 3)
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def record_split_result(module_dir: Path, record: SplitRecord) -> None:
+    """Write a split execution record to .meta/split-feedback.json.
+
+    Args:
+        module_dir: Module documentation directory
+        record: SplitRecord with execution metrics
+    """
+    meta_dir = module_dir / ".meta"
+    meta_dir.mkdir(parents=True, exist_ok=True)
+    feedback_path = meta_dir / FEEDBACK_FILENAME
+
+    records = _load_records(feedback_path)
+
+    # Serialize record
+    entry = asdict(record)
+    entry["composite_score"] = record.composite_score
+    records.append(entry)
+
+    # Keep only recent records
+    records = records[-MAX_RECORDS:]
+
+    try:
+        feedback_path.write_text(
+            json.dumps(records, ensure_ascii=False, indent=2),
+            encoding="utf-8",
+        )
+    except OSError as e:
+        logger.warning("Failed to write split feedback: %s", e)
+
+
+def get_best_resolution(
+    module_dir: Path,
+    doc_type: str,
+    default_resolution: float = 1.0,
+) -> dict:
+    """Read history and return best resolution parameter.
+
+    Simplified: just reuse the last successful resolution parameter.
+    No weighted scoring — if the last run succeeded, use its parameters.
+
+    Args:
+        module_dir: Module documentation directory
+        doc_type: Document type to filter by
+        default_resolution: Default when no history exists
+
+    Returns:
+        {"resolution": float, "records": int, "recommendation": str}
+    """
+    feedback_path = module_dir / ".meta" / FEEDBACK_FILENAME
+    records = _load_records(feedback_path)
+
+    relevant = [r for r in records if r.get("doc_type") == doc_type]
+
+    if not relevant:
+        return {
+            "resolution": default_resolution,
+            "records": 0,
+            "recommendation": "no-history",
+        }
+
+    # Simple: use the last successful record's resolution
+    last = relevant[-1]
+    return {
+        "resolution": last.get("resolution", default_resolution),
+        "records": len(relevant),
+        "recommendation": "use-last-successful",
+    }
+
+
+def get_split_history(
+    module_dir: Path,
+    doc_type: str | None = None,
+) -> list[dict]:
+    """Get split history records for debugging and analysis.
+
+    Args:
+        module_dir: Module documentation directory
+        doc_type: Optional filter
+
+    Returns:
+        List of historical records.
+    """
+    feedback_path = module_dir / ".meta" / FEEDBACK_FILENAME
+    records = _load_records(feedback_path)
+    if doc_type:
+        records = [r for r in records if r.get("doc_type") == doc_type]
+    return records
+
+
+# ---------------------------------------------------------------------------
+# Internal
+# ---------------------------------------------------------------------------
+
+
+def _load_records(path: Path) -> list[dict]:
+    """Load feedback records file."""
+    if not path.exists():
+        return []
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+        return data if isinstance(data, list) else []
+    except (json.JSONDecodeError, OSError):
+        return []

core/skeleton/split_plan.py

@@ -0,0 +1,219 @@
+"""Split planning — orchestrate split strategy selection.
+
+Implements the priority chain for split planning:
+  1. Cache hit (skeleton hash unchanged) — instant
+  2. Community detection (dependency graph, zero LLM cost)
+  3. LLM-assisted grouping (business-domain, optional)
+  4. Package-based grouping (code rules, zero LLM cost)
+  5. Simple split (equal file count, last resort)
+
+All thresholds loaded from SplitConfig (yaml-driven), no hardcoded numbers.
+
+Usage:
+    from core.skeleton.split_plan import plan_splits
+
+    plan = plan_splits(entries, file_list, split_config, dep_graph)
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any
+
+from core.skeleton.split import SplitConfig, SplitPlan
+from core.skeleton.split_plan_helpers import (
+    make_split, derive_name, balanced_split,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def plan_splits(
+    entries: list[dict[str, Any]],
+    file_list: list[dict[str, Any]],
+    split_config: SplitConfig,
+    dep_graph: Any | None = None,
+    doc_type: str = "business-logic",
+    module_dir: Path | None = None,
+    *,
+    llm_strategy: Any | None = None,
+    is_delegated: bool = False,
+) -> SplitPlan:
+    """Main entry point for split planning with priority chain.
+
+    Priority chain: Cache -> Community -> LLM -> Package -> Simple
+
+    Args:
+        llm_strategy: Optional LlmStrategy instance for LLM-assisted splitting.
+                      Injected by engine/ caller. None = skip LLM strategy.
+        is_delegated: Whether running in delegated mode (Agent handles grouping).
+    """
+    if not file_list:
+        return SplitPlan(splits=[], strategy="empty")
+
+    max_lines = split_config.effective_max_lines(doc_type)
+    max_files = split_config.max_files_per_shard
+    total_files = len(file_list)
+    total_lines = sum(f.get("lines", 0) for f in file_list)
+
+    if total_files <= max_files and total_lines <= max_lines:
+        logger.debug("split not needed for doc_type=%s (files=%d, lines=%d)", doc_type, total_files, total_lines)
+        return SplitPlan(
+            splits=[make_split("all", file_list)],
+            strategy="single",
+        )
+
+    # Strategy 1: Cache hit
+    if module_dir:
+        cached = _try_cache(module_dir, doc_type)
+        if cached:
+            logger.info("split strategy=cache doc_type=%s splits=%d", doc_type, len(cached.splits))
+            return cached
+
+    # Strategy 2: Community detection
+    plan = _try_community(file_list, entries, split_config, dep_graph, doc_type)
+    if plan:
+        logger.info("split strategy=community doc_type=%s splits=%d", doc_type, len(plan.splits))
+        _save_to_cache(module_dir, doc_type, plan)
+        return plan
+    logger.debug("community detection skipped for doc_type=%s", doc_type)
+
+    # Strategy 3: LLM-assisted grouping
+    from core.skeleton.split_plan_llm import try_llm_split
+    plan = try_llm_split(file_list, split_config, doc_type, module_dir,
+                         strategy=llm_strategy, is_delegated=is_delegated)
+    if plan:
+        logger.info("split strategy=llm doc_type=%s splits=%d", doc_type, len(plan.splits))
+        if plan.strategy != "agent-pending":
+            _save_to_cache(module_dir, doc_type, plan)
+        return plan
+    logger.debug("llm split skipped for doc_type=%s", doc_type)
+
+    # Strategy 4: Package-based grouping
+    from core.skeleton.split_plan_helpers import try_package
+    plan = try_package(file_list, split_config, doc_type)
+    if plan:
+        logger.info("split strategy=package doc_type=%s splits=%d", doc_type, len(plan.splits))
+        _save_to_cache(module_dir, doc_type, plan)
+        return plan
+    logger.debug("package split skipped for doc_type=%s", doc_type)
+
+    # Strategy 5: Simple split (last resort)
+    logger.info("split strategy=simple (last resort) doc_type=%s files=%d", doc_type, total_files)
+    plan = _simple_split(file_list, split_config, doc_type)
+    _save_to_cache(module_dir, doc_type, plan)
+    return plan
+
+
+# ---------------------------------------------------------------------------
+# Strategy 1: Cache
+# ---------------------------------------------------------------------------
+
+
+def _try_cache(module_dir: Path, doc_type: str) -> SplitPlan | None:
+    """Try to load a cached split plan."""
+    try:
+        from core.skeleton.split_cache import compute_skeleton_hash, get_cached_plan
+        current_hash = compute_skeleton_hash(module_dir)
+        cached = get_cached_plan(module_dir, doc_type, current_hash)
+        if cached and "splits" in cached:
+            return SplitPlan(
+                splits=cached["splits"],
+                strategy="cache",
+            )
+    except Exception as e:
+        logger.debug("Split cache load failed for %s: %s", doc_type, e)
+    return None
+
+
+def _save_to_cache(module_dir: Path | None, doc_type: str, plan: SplitPlan) -> None:
+    """Persist split plan to cache for future reuse."""
+    if not module_dir or not plan.splits:
+        return
+    try:
+        from core.skeleton.split_cache import compute_skeleton_hash, save_plan_cache
+        current_hash = compute_skeleton_hash(module_dir)
+        save_plan_cache(
+            module_dir, current_hash, doc_type,
+            {"splits": plan.splits},
+            num_splits=len(plan.splits),
+        )
+    except Exception as e:
+        logger.debug("Failed to save split cache for %s: %s", doc_type, e)
+
+
+# ---------------------------------------------------------------------------
+# Strategy 2: Community detection
+# ---------------------------------------------------------------------------
+
+
+def _try_community(
+    file_list: list[dict],
+    entries: list[dict],
+    split_config: SplitConfig,
+    dep_graph: Any | None,
+    doc_type: str,
+) -> SplitPlan | None:
+    """Try community detection split."""
+    try:
+        from core.skeleton.dependency_graph import DependencyGraph, build_dependency_graph
+        from core.skeleton.community import split_by_community
+    except ImportError:
+        return None
+
+    if dep_graph is None:
+        if not entries:
+            return None
+        file_basenames = {Path(f.get("name", "")).name for f in file_list}
+        relevant = [e for e in entries if Path(e.get("file", "")).name in file_basenames]
+        if not relevant:
+            return None
+        dep_graph = build_dependency_graph(relevant)
+
+    if not isinstance(dep_graph, DependencyGraph) or not dep_graph.adjacency:
+        return None
+
+    max_lines = split_config.effective_max_lines(doc_type)
+    max_files = split_config.max_files_per_shard
+
+    groups = split_by_community(
+        file_list, dep_graph,
+        max_files_per_shard=max_files,
+        max_lines_per_shard=max_lines,
+    )
+
+    if not groups or len(groups) <= 1:
+        return None
+
+    total_files = len(file_list)
+    max_reasonable_splits = max(6, total_files // max_files + 2)
+    if len(groups) > max_reasonable_splits:
+        return None
+
+    splits = [make_split(derive_name(g, noise_words=split_config.noise_words), g) for g in groups]
+    return SplitPlan(splits=splits, strategy="community")
+
+
+# ---------------------------------------------------------------------------
+# Strategy 5: Simple split
+# ---------------------------------------------------------------------------
+
+
+def _simple_split(
+    file_list: list[dict],
+    split_config: SplitConfig,
+    doc_type: str,
+) -> SplitPlan:
+    """Last resort: equal-count split."""
+    max_lines = split_config.effective_max_lines(doc_type)
+    max_files = split_config.max_files_per_shard
+
+    chunks = balanced_split(file_list, max_lines, max_files)
+    splits = [make_split(f"group-{i+1}", c) for i, c in enumerate(chunks)]
+    return SplitPlan(splits=splits, strategy="simple")

core/skeleton/split_plan_helpers.py

@@ -0,0 +1,305 @@
+"""Split plan helpers — shared utilities for split strategies.
+
+Includes:
+- Package-based grouping strategy
+- Name derivation (semantic business-domain naming)
+- Balanced splitting (LPT greedy)
+- Small-split merging
+"""
+
+from __future__ import annotations
+
+import math
+import os
+import re
+from collections import Counter, defaultdict
+from typing import Any
+
+
+def make_split(name: str, files: list[dict]) -> dict[str, Any]:
+    return {
+        "name": name,
+        "files": [f.get("rel_path", f.get("name", "")) for f in files],
+        "file_count": len(files),
+        "lines": sum(f.get("lines", 0) for f in files),
+        "packages": list({f.get("package", "") for f in files}),
+    }
+
+
+def derive_name(
+    files: list[dict],
+    split_name_suffixes: tuple[str, ...] | None = None,
+    noise_words: frozenset[str] | None = None,
+) -> str:
+    """Derive a semantic business-domain name from file list.
+
+    Strategy priority:
+    1. Common sub-package beyond base (e.g., service.group -> "group")
+    2. Dominant business keywords from class names, combined for specificity
+    3. Common prefix of stems (fallback)
+    """
+    if not files:
+        return "group"
+
+    suffixes = split_name_suffixes or (
+        "ServiceImpl", "Service", "Handler", "Processor",
+        "Manager", "Controller", "Listener", "Client", "Biz", "BizImpl",
+    )
+
+    packages = [f.get("package", "") for f in files if f.get("package")]
+    if packages:
+        parts_list = [p.split(".") for p in packages]
+        if parts_list:
+            min_len = min(len(p) for p in parts_list)
+            common_depth = 0
+            for i in range(min_len):
+                if len(set(p[i] for p in parts_list)) == 1:
+                    common_depth = i + 1
+                else:
+                    break
+            diverging = Counter()
+            for parts in parts_list:
+                if len(parts) > common_depth:
+                    seg = parts[common_depth]
+                    if seg not in ("impl", "service", "base", "controller", "config", "util", "common"):
+                        diverging[seg] += 1
+            if diverging:
+                top_pkg, top_count = diverging.most_common(1)[0]
+                if top_count >= len(files) * 0.5:
+                    return top_pkg
+
+    keywords = Counter()
+    noise = noise_words or frozenset({
+        "service", "impl", "base", "controller", "manager",
+        "handler", "listener", "api", "java", "abstract", "default", "i",
+    })
+    for f in files:
+        raw_name = f.get("name", "")
+        basename = raw_name.rsplit("/", 1)[-1].rsplit("\\", 1)[-1].rsplit(".", 1)[0]
+        words = re.findall(r'[A-Z][a-z]+', basename)
+        for w in words:
+            w_lower = w.lower()
+            if w_lower not in noise and len(w_lower) >= 3:
+                keywords[w_lower] += 1
+
+    if keywords:
+        top_keywords = keywords.most_common(5)
+        selected = []
+        for k, c in top_keywords:
+            if c >= max(2, len(files) * 0.2):
+                selected.append(k)
+            if len(selected) >= 2:
+                break
+        if selected:
+            return "-".join(selected)
+        elif top_keywords:
+            return top_keywords[0][0]
+
+    stems: list[str] = []
+    for f in files:
+        name = f.get("name", "").rsplit(".", 1)[0]
+        for s in sorted(suffixes, key=len, reverse=True):
+            if name.endswith(s) and len(name) > len(s):
+                name = name[:-len(s)]
+                break
+        stems.append(name)
+    prefix = os.path.commonprefix(stems)
+    if len(prefix) >= 4:
+        return re.sub(r'([a-z])([A-Z])', r'\1-\2', prefix).lower().strip("-")
+    if files:
+        largest = max(files, key=lambda f: f.get("lines", 0))
+        return largest.get("name", "group").rsplit(".", 1)[0].lower()[:30]
+    return "group"
+
+
+def derive_name_with_context(
+    files: list[dict],
+    pkg_label: str,
+    chunk_idx: int,
+    total_chunks: int,
+    noise_words: frozenset[str] | None = None,
+) -> str:
+    """Derive a name for a chunk within a larger package group."""
+    noise = set(noise_words or {
+        "service", "impl", "base", "controller", "manager",
+        "handler", "listener", "api", "java", "abstract", "default", "i",
+    })
+    noise.add(pkg_label.lower())
+
+    keywords = Counter()
+    for f in files:
+        basename = f.get("name", "").rsplit("/", 1)[-1].rsplit("\\", 1)[-1].rsplit(".", 1)[0]
+        words = re.findall(r'[A-Z][a-z]+', basename)
+        for w in words:
+            w_lower = w.lower()
+            if w_lower not in noise and len(w_lower) >= 3:
+                keywords[w_lower] += 1
+
+    if keywords:
+        top = keywords.most_common(2)
+        suffix = "-".join(k for k, _ in top)
+        return f"{pkg_label}-{suffix}"
+
+    return f"{pkg_label}-{chunk_idx}"
+
+
+def rebalance_groups(groups: list[list[dict]], max_files: int) -> None:
+    """Split oversized groups to respect max_files_per_shard constraint."""
+    i = 0
+    while i < len(groups):
+        if len(groups[i]) > max_files:
+            overflow = groups[i][max_files:]
+            groups[i] = groups[i][:max_files]
+            while overflow:
+                chunk = overflow[:max_files]
+                overflow = overflow[max_files:]
+                groups.append(chunk)
+        i += 1
+
+
+def deduplicate_names(splits: list[dict]) -> None:
+    """Ensure all split names are unique by appending a counter to duplicates."""
+    name_counts = Counter(s["name"] for s in splits)
+    dupes = {name for name, count in name_counts.items() if count > 1}
+    if not dupes:
+        return
+    counters: dict[str, int] = {}
+    for s in splits:
+        if s["name"] in dupes:
+            counters.setdefault(s["name"], 0)
+            counters[s["name"]] += 1
+            s["name"] = f"{s['name']}-{counters[s['name']]}"
+
+
+def group_by_package(files: list[dict], depth: int = 3) -> dict[str, list[dict]]:
+    groups: dict[str, list[dict]] = defaultdict(list)
+    for f in files:
+        parts = f.get("package", "").split(".")
+        key = ".".join(parts[:depth]) if len(parts) >= depth else f.get("package", "root")
+        groups[key].append(f)
+    return dict(groups)
+
+
+def balanced_split(files: list[dict], max_lines: int, max_files: int) -> list[list[dict]]:
+    """LPT greedy balanced split."""
+    total_lines = sum(f.get("lines", 0) for f in files)
+    n = max(math.ceil(len(files) / max_files), math.ceil(total_lines / max_lines), 2)
+    buckets: list[list[dict]] = [[] for _ in range(n)]
+    bucket_lines = [0] * n
+    for f in sorted(files, key=lambda x: -x.get("lines", 0)):
+        lightest = min(range(n), key=lambda i: bucket_lines[i])
+        buckets[lightest].append(f)
+        bucket_lines[lightest] += f.get("lines", 0)
+    return [b for b in buckets if b]
+
+
+def merge_small(
+    splits: list[dict],
+    max_lines: int,
+    max_files: int,
+    merge_ratio: float = 0.25,
+    file_list: list[dict] | None = None,
+) -> list[dict]:
+    """Merge small splits into neighbors when they're far below target size."""
+    small_line_threshold = max(200, int(max_lines * merge_ratio))
+    small_file_threshold = max(5, int(max_files * merge_ratio))
+
+    file_lines_lookup: dict[str, int] = {}
+    if file_list:
+        for f in file_list:
+            file_lines_lookup[f.get("name", "")] = f.get("lines", 0)
+
+    large: list[dict] = []
+    small_files: list[dict] = []
+    for s in splits:
+        if s["lines"] < small_line_threshold and s["file_count"] < small_file_threshold:
+            avg_lines = s["lines"] // max(s["file_count"], 1)
+            for fname in s["files"]:
+                actual_lines = file_lines_lookup.get(fname, avg_lines)
+                small_files.append({"name": fname, "lines": actual_lines, "package": ""})
+        else:
+            large.append(s)
+
+    if small_files and large:
+        for f in small_files:
+            lightest = min(large, key=lambda x: x["lines"])
+            if lightest["lines"] + f["lines"] <= max_lines and lightest["file_count"] + 1 <= max_files:
+                lightest["files"].append(f["name"])
+                lightest["file_count"] = len(lightest["files"])
+                lightest["lines"] += f["lines"]
+            else:
+                large.append(make_split("misc", [f]))
+    elif small_files:
+        large.append(make_split("misc", small_files))
+    return large
+
+
+def package_label(pkg: str, depth: int) -> str:
+    """Extract a meaningful label from a package path."""
+    parts = pkg.split(".")
+    generic = {
+        "impl", "service", "base", "controller", "config", "util", "common", "api",
+        "model", "entity", "manager", "provider", "client", "listener", "event",
+        "promotion", "app", "example",
+    }
+    meaningful = [p for p in parts[depth:] if p not in generic]
+    if meaningful:
+        return "-".join(meaningful)
+    for p in reversed(parts):
+        if p not in generic and len(p) > 3:
+            return p
+    return parts[-1] if parts else "group"
+
+
+def try_package(
+    file_list: list[dict],
+    split_config: Any,
+    doc_type: str,
+) -> Any | None:
+    """Try package-based grouping."""
+    from core.skeleton.split import SplitPlan
+
+    max_lines = split_config.effective_max_lines(doc_type)
+    override = split_config.per_doc_type_overrides.get(doc_type, {})
+    max_files = override.get("max_files_per_shard", split_config.max_files_per_shard)
+
+    best_groups = None
+    best_depth = 3
+    for depth in range(3, 10):
+        groups = group_by_package(file_list, depth=depth)
+        if len(groups) > 1:
+            best_groups = groups
+            best_depth = depth
+            break
+
+    if best_groups is None or len(best_groups) <= 1:
+        return None
+
+    splits: list[dict] = []
+    warnings: list[str] = []
+
+    for pkg, pkg_files in best_groups.items():
+        pkg_lbl = package_label(pkg, best_depth)
+        pkg_lines = sum(f.get("lines", 0) for f in pkg_files)
+        if len(pkg_files) > max_files or pkg_lines > max_lines:
+            chunks = balanced_split(pkg_files, max_lines, max_files)
+            for i, c in enumerate(chunks):
+                chunk_name = derive_name_with_context(
+                    c, pkg_lbl, i + 1, len(chunks),
+                    noise_words=split_config.noise_words,
+                )
+                splits.append(make_split(chunk_name, c))
+            warnings.append(f"Package {pkg} oversized, split into {len(chunks)} groups")
+        else:
+            splits.append(make_split(pkg_lbl, pkg_files))
+
+    splits = merge_small(
+        splits, max_lines, max_files,
+        split_config.merge_threshold_ratio, file_list=file_list,
+    )
+
+    if len(splits) <= 1:
+        return None
+
+    deduplicate_names(splits)
+    return SplitPlan(splits=splits, strategy="package", warnings=warnings)