source-kb 0.2.2__py3-none-any.whl

core/skeleton/query.py

@@ -0,0 +1,195 @@
+"""Skeleton query tool — stats, search, and high-complexity method listing.
+
+Provides programmatic access to skeleton data for diagnostics.
+Migrated from engine/skeleton/query_skeleton.py.
+
+No CLI I/O in this module — returns structured data only.
+CLI layer handles formatting and output.
+
+Usage:
+    from core.skeleton.query import stats, search, high_methods
+    result = stats(entries)
+    matches = search(entries, "OrderService")
+    methods = high_methods(entries)
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+
+# ---------------------------------------------------------------------------
+# Data loading
+# ---------------------------------------------------------------------------
+
+
+def load_skeleton(path: Path) -> list[dict]:
+    """Load skeleton JSON, supporting both bare list and dict wrapper formats.
+
+    Args:
+        path: Path to skeleton JSON file or directory of shard files.
+
+    Returns:
+        List of skeleton entries.
+
+    Raises:
+        FileNotFoundError: If path does not exist.
+        json.JSONDecodeError: If JSON is malformed.
+    """
+    if path.is_dir():
+        return _load_from_directory(path)
+
+    raw = json.loads(path.read_text(encoding="utf-8"))
+    if isinstance(raw, dict):
+        return raw.get("data", raw.get("files", []))
+    if isinstance(raw, list):
+        return raw
+    return []
+
+
+def _load_from_directory(directory: Path) -> list[dict]:
+    """Load and merge skeleton entries from a directory of shard files."""
+    entries: list[dict] = []
+    for f in sorted(directory.glob("*.json")):
+        try:
+            data = json.loads(f.read_text(encoding="utf-8"))
+            if isinstance(data, list):
+                entries.extend(data)
+            elif isinstance(data, dict):
+                entries.extend(data.get("data", data.get("files", [])))
+        except (json.JSONDecodeError, OSError):
+            pass
+    return entries
+
+
+# ---------------------------------------------------------------------------
+# Stats subcommand
+# ---------------------------------------------------------------------------
+
+
+def stats(entries: list[dict]) -> dict[str, Any]:
+    """Compute skeleton statistics.
+
+    Returns:
+        {
+            "files": int,
+            "total_lines": int,
+            "methods": int,
+            "complexity": {"high": n, "medium": n, "low": n},
+        }
+    """
+    total_files = len(entries)
+    total_methods = 0
+    total_lines = 0
+    complexity_counts: dict[str, int] = {"high": 0, "medium": 0, "low": 0}
+
+    for entry in entries:
+        total_lines += entry.get("total_lines", 0)
+        for m in entry.get("methods", []):
+            total_methods += 1
+            c = m.get("complexity", "unknown")
+            if c in complexity_counts:
+                complexity_counts[c] += 1
+
+    return {
+        "files": total_files,
+        "total_lines": total_lines,
+        "methods": total_methods,
+        "complexity": {k: v for k, v in complexity_counts.items() if v > 0},
+    }
+
+
+# ---------------------------------------------------------------------------
+# High-methods subcommand
+# ---------------------------------------------------------------------------
+
+
+def high_methods(entries: list[dict]) -> list[dict[str, Any]]:
+    """List all high-complexity methods.
+
+    Returns:
+        List of {"file": str, "method": str, "line_count": int}
+    """
+    results: list[dict[str, Any]] = []
+    for entry in entries:
+        filepath = entry.get("file", "")
+        for m in entry.get("methods", []):
+            if m.get("complexity") == "high":
+                results.append({
+                    "file": filepath,
+                    "method": m.get("name", "?"),
+                    "line_count": m.get("line_count", 0),
+                })
+    return results
+
+
+# ---------------------------------------------------------------------------
+# Search subcommand (fuzzy match)
+# ---------------------------------------------------------------------------
+
+
+def search(entries: list[dict], query: str) -> list[dict[str, Any]]:
+    """Fuzzy search for methods or classes matching query.
+
+    Performs case-insensitive substring match on class names, method names,
+    and file paths.
+
+    Returns:
+        List of {"file": str, "type": "class"|"method", "name": str, "line": int}
+    """
+    results: list[dict[str, Any]] = []
+    pattern = re.compile(re.escape(query), re.IGNORECASE)
+
+    for entry in entries:
+        filepath = entry.get("file", "")
+
+        # Search in file path
+        if pattern.search(filepath):
+            results.append({
+                "file": filepath,
+                "type": "file",
+                "name": filepath,
+                "line": 0,
+            })
+
+        # Search in class names
+        for cls in entry.get("classes", []):
+            name = cls.get("name", "")
+            if name and pattern.search(name):
+                results.append({
+                    "file": filepath,
+                    "type": "class",
+                    "name": name,
+                    "line": cls.get("line", 0),
+                })
+
+        # Search in method names
+        for m in entry.get("methods", []):
+            name = m.get("name", "")
+            if name and pattern.search(name):
+                results.append({
+                    "file": filepath,
+                    "type": "method",
+                    "name": name,
+                    "line": m.get("line", 0),
+                })
+
+    return results
+
+
+# ---------------------------------------------------------------------------
+# File list and method count utilities
+# ---------------------------------------------------------------------------
+
+
+def file_list(entries: list[dict]) -> list[str]:
+    """Extract all file paths from skeleton entries."""
+    return [entry.get("file", "") for entry in entries if entry.get("file")]
+
+
+def method_count(entries: list[dict]) -> int:
+    """Count total methods across all entries."""
+    return sum(len(e.get("methods", [])) for e in entries)

core/skeleton/shard_context.py

@@ -0,0 +1,177 @@
+"""Shard context module — builds context for N+1 shard from completed shards.
+
+Provides prior shard summaries and neighbor file lists to avoid information
+silos between shards of the same doc type.
+
+No CLI I/O, no LLM calls. Pure file reading and formatting.
+
+Usage:
+    from core.skeleton.shard_context import build_shard_context
+
+    context = build_shard_context(
+        module_dir=Path("knowledge/point/point-base"),
+        doc_type="business-logic",
+        current_shard_name="payment",
+        completed_shards=["order", "inventory"],
+    )
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def build_shard_context(
+    module_dir: Path,
+    doc_type: str,
+    current_shard_name: str,
+    completed_shards: list[str] | None = None,
+    max_chars: int = 1500,
+    preset: dict | None = None,
+) -> str:
+    """Build context summary from prior shards of the same doc type.
+
+    Args:
+        module_dir: Module knowledge directory
+        doc_type: Document type (e.g., "business-logic")
+        current_shard_name: Current shard name (excluded from context)
+        completed_shards: List of completed shard names (in completion order)
+        max_chars: Maximum injected character count
+        preset: Preset configuration dict (for doc_type → filename resolution)
+
+    Returns:
+        Formatted context text for prompt injection
+    """
+    if not completed_shards:
+        return ""
+
+    doc_basename = _doc_type_to_filename(doc_type, preset=preset).replace(".md", "")
+    shard_summaries: list[str] = []
+
+    for shard_name in completed_shards:
+        if shard_name == current_shard_name:
+            continue
+
+        # Try multiple naming patterns
+        candidates = [
+            module_dir / f"{doc_basename}-{shard_name}.md",
+            module_dir / f".shard-{doc_basename}-{shard_name}.md",
+        ]
+
+        for path in candidates:
+            if path.exists():
+                summary = _extract_shard_summary(path)
+                if summary:
+                    shard_summaries.append(f"### {shard_name}\n{summary}")
+                break
+
+    if not shard_summaries:
+        return ""
+
+    header = "## Content already covered by prior shards (avoid duplication)\n\n"
+    body = "\n\n".join(shard_summaries)
+    full_text = header + body
+
+    if len(full_text) > max_chars:
+        full_text = full_text[:max_chars - 30] + "\n\n[Context truncated]"
+
+    return full_text
+
+
+def build_neighbor_file_list(
+    module_dir: Path,
+    doc_type: str,
+    current_shard_name: str,
+    all_shards: list[str],
+) -> str:
+    """Build neighbor shard file list summary for cross-shard awareness.
+
+    Args:
+        module_dir: Module knowledge directory
+        doc_type: Document type
+        current_shard_name: Current shard name
+        all_shards: All shard names in the plan
+
+    Returns:
+        Formatted neighbor file list text
+    """
+    from core.paths import file_list_dir
+
+    fl_dir = file_list_dir(module_dir)
+    neighbors: list[str] = []
+
+    for shard_name in all_shards:
+        if shard_name == current_shard_name:
+            continue
+
+        fl_path = fl_dir / f"{doc_type}-{shard_name}.txt"
+        if not fl_path.exists():
+            continue
+
+        try:
+            lines = fl_path.read_text(encoding="utf-8").splitlines()
+            file_count = sum(1 for l in lines if l.strip())
+            # Show first few files as preview
+            preview = [l.strip().rsplit("/", 1)[-1] for l in lines[:3] if l.strip()]
+            preview_str = ", ".join(preview)
+            if file_count > 3:
+                preview_str += f" ... (+{file_count - 3})"
+            neighbors.append(f"- **{shard_name}** ({file_count} files): {preview_str}")
+        except OSError:
+            continue
+
+    if not neighbors:
+        return ""
+
+    return "### Neighbor shard file lists\n\n" + "\n".join(neighbors)
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _extract_shard_summary(path: Path, max_lines: int = 12) -> str:
+    """Extract shard document summary: headings + first few content lines."""
+    try:
+        content = path.read_text(encoding="utf-8")
+    except (OSError, UnicodeDecodeError):
+        return ""
+
+    lines = content.splitlines()
+    if not lines:
+        return ""
+
+    # Extract ## headings
+    headings = [l.strip() for l in lines if l.strip().startswith("## ")]
+
+    # Take first N non-empty lines as summary
+    summary_lines: list[str] = []
+    for line in lines[:max_lines]:
+        stripped = line.strip()
+        if stripped:
+            summary_lines.append(stripped)
+
+    result_parts: list[str] = []
+    if summary_lines:
+        result_parts.append("\n".join(summary_lines[:5]))
+    if headings:
+        result_parts.append("Sections: " + " | ".join(h.lstrip("#").strip() for h in headings[:8]))
+
+    return "\n".join(result_parts)
+
+
+def _doc_type_to_filename(doc_type: str, preset: dict | None = None) -> str:
+    """Map doc_type key to filename using preset config."""
+    if preset:
+        from core.preset import get_doc_filename
+        filename = get_doc_filename(preset, doc_type, strict=False)
+        if filename:
+            return filename
+    # Fallback: assume doc_type matches filename pattern
+    return f"{doc_type}.md"

core/skeleton/split.py

@@ -0,0 +1,180 @@
+"""Split configuration and shard count computation.
+
+Provides SplitConfig (all thresholds from yaml) and compute_splits (shard count).
+Split *planning* (files → shards assignment) lives in split_plan.py.
+
+Usage:
+    from core.skeleton.split import SplitConfig, SplitPlan, compute_splits
+
+    config = SplitConfig.from_preset(preset, mode="readwrite")
+    n = compute_splits(config, total_bytes=500000, total_lines=12000)
+"""
+
+from __future__ import annotations
+
+import logging
+import math
+from dataclasses import dataclass, field
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# SplitConfig — all thresholds from configuration
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class SplitConfig:
+    """Encapsulates all split threshold parameters. Loaded from preset/config yaml."""
+
+    max_bytes: int = 300 * 1024
+    max_lines: int = 8000
+    max_files_per_shard: int = 80
+    llm_sample_limit: int = 200
+    merge_threshold_ratio: float = 0.25  # Splits below this fraction of max are merged
+    skip_file_count_doc_types: frozenset[str] = frozenset({"source-tree-analysis", "index"})
+    per_doc_type_overrides: dict[str, dict[str, int]] = None  # type: ignore[assignment]
+    noise_words: frozenset[str] = frozenset({
+        "service", "impl", "base", "controller", "manager",
+        "handler", "listener", "api", "java", "abstract", "default", "i",
+    })
+
+    def __post_init__(self):
+        if self.per_doc_type_overrides is None:
+            object.__setattr__(self, "per_doc_type_overrides", {})
+
+    @classmethod
+    def from_preset(cls, preset: dict, mode: str = "output-only", project_split: dict | None = None) -> SplitConfig:
+        """Load from preset config with mode-specific defaults and per-doc-type overrides.
+
+        Args:
+            preset: Preset configuration dict (from doc_types.yaml)
+            mode: Execution mode ("readwrite" or "output-only")
+            project_split: Optional project-level split overrides (from kb-project.yaml)
+                          These take precedence over preset defaults.
+        """
+        split_cfg = preset.get("split", {}).get(mode, {})
+        defaults = _MODE_DEFAULTS[mode]
+
+        # Project-level overrides (kb-project.yaml > preset defaults)
+        project_mode_cfg = {}
+        if project_split:
+            project_mode_cfg = project_split.get(mode, project_split)
+
+        # Collect per-doc-type overrides from doc_types section
+        doc_types = preset.get("doc_types", {})
+        overrides: dict[str, dict[str, int]] = {}
+        for dt_key, dt_config in doc_types.items():
+            if isinstance(dt_config, dict) and "split_override" in dt_config:
+                mode_override = dt_config["split_override"].get(mode, {})
+                if mode_override:
+                    overrides[dt_key] = mode_override
+
+        # Determine global_view doc types (skip file count limit)
+        from core.preset import get_global_view_types
+        global_view_types = get_global_view_types(preset)
+
+        # Resolve values: project > preset > mode defaults
+        def _resolve(key: str, default: int | float):
+            if key in project_mode_cfg:
+                return project_mode_cfg[key]
+            return split_cfg.get(key, default)
+
+        return cls(
+            max_bytes=_resolve("max_bytes", defaults["max_bytes"]),
+            max_lines=_resolve("max_lines", defaults["max_lines"]),
+            max_files_per_shard=_resolve("max_files_per_shard", defaults["max_files_per_shard"]),
+            llm_sample_limit=_resolve("llm_sample_limit", defaults["llm_sample_limit"]),
+            merge_threshold_ratio=_resolve("merge_threshold_ratio", 0.25),
+            skip_file_count_doc_types=global_view_types or frozenset({"source-tree-analysis", "index"}),
+            per_doc_type_overrides=overrides,
+            noise_words=frozenset(preset.get("split", {}).get("noise_words", [])) or SplitConfig.noise_words,
+        )
+
+    def effective_max_lines(self, doc_type: str) -> int:
+        """Get max_lines for a specific doc type (respects per-doc-type overrides)."""
+        override = self.per_doc_type_overrides.get(doc_type, {})
+        return override.get("max_lines", self.max_lines)
+
+    def effective_max_bytes(self, doc_type: str) -> int:
+        """Get max_bytes for a specific doc type (respects per-doc-type overrides)."""
+        override = self.per_doc_type_overrides.get(doc_type, {})
+        return override.get("max_bytes", self.max_bytes)
+
+
+_MODE_DEFAULTS: dict[str, dict[str, int]] = {
+    "output-only": {"max_bytes": 300 * 1024, "max_lines": 8000, "max_files_per_shard": 80, "llm_sample_limit": 200},
+    "readwrite": {"max_bytes": 500 * 1024, "max_lines": 10000, "max_files_per_shard": 80, "llm_sample_limit": 200},
+}
+
+
+# ---------------------------------------------------------------------------
+# compute_splits — determine number of shards needed
+# ---------------------------------------------------------------------------
+
+
+def compute_splits(
+    config: SplitConfig,
+    *,
+    total_bytes: int = 0,
+    total_lines: int = 0,
+    file_count: int = 0,
+    doc_type: str = "",
+) -> int:
+    """Compute number of splits needed based on input size.
+
+    Priority: bytes > lines > file_count (file count is upper-bound protection only).
+    """
+    if file_count <= 0 and total_lines <= 0 and total_bytes <= 0:
+        return 1
+
+    max_lines = config.effective_max_lines(doc_type)
+    max_bytes = config.effective_max_bytes(doc_type)
+
+    # Per-doc-type max_files override
+    override = config.per_doc_type_overrides.get(doc_type, {})
+    max_files = override.get("max_files_per_shard", config.max_files_per_shard)
+
+    # Priority 1: bytes
+    splits_by_bytes = math.ceil(total_bytes / max_bytes) if total_bytes > 0 else 1
+    # Priority 2: lines
+    splits_by_lines = math.ceil(total_lines / max_lines) if total_lines > 0 else 1
+    # Take the larger
+    splits = max(splits_by_bytes, splits_by_lines)
+
+    # Priority 3: file count upper-bound protection
+    if file_count > 0 and splits > 0 and doc_type not in config.skip_file_count_doc_types:
+        files_per_shard = math.ceil(file_count / splits)
+        if files_per_shard > max_files:
+            splits = math.ceil(file_count / max_files)
+
+    return max(1, splits)
+
+
+# ---------------------------------------------------------------------------
+# SplitPlan — result dataclass (used by split_plan.py)
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class SplitPlan:
+    """Result of split planning."""
+    splits: list[dict[str, Any]] = field(default_factory=list)
+    strategy: str = "single"
+    warnings: list[str] = field(default_factory=list)
+    pending_agent_grouping: bool = False
+    grouping_request_path: str | None = None
+
+    @property
+    def recommended_agents(self) -> int:
+        return len(self.splits)
+
+
+# Cache functions re-exported from split_cache module
+from core.skeleton.split_cache import (  # noqa: E402, F401
+    compute_skeleton_hash,
+    get_cached_plan,
+    save_plan_cache,
+)

core/skeleton/split_cache.py

@@ -0,0 +1,107 @@
+"""Split plan caching — persist and retrieve split decisions.
+
+Caches split plans keyed by skeleton hash to avoid recomputing when source
+hasn't changed. Supports hysteresis by providing previous split counts.
+
+Usage:
+    from core.skeleton.split_cache import compute_skeleton_hash, get_cached_plan, save_plan_cache
+
+    hash = compute_skeleton_hash(module_dir)
+    cached = get_cached_plan(module_dir, "business-logic", hash)
+    if not cached:
+        plan = compute_new_plan(...)
+        save_plan_cache(module_dir, hash, "business-logic", plan)
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+_CACHE_FILE = "split-plan-cache.json"
+
+
+def compute_skeleton_hash(module_dir: Path) -> str:
+    """SHA-256 hash of skeleton + file-lists (for cache invalidation)."""
+    from core.paths import resolve_skeleton, file_list_dir
+
+    h = hashlib.sha256()
+    skel = resolve_skeleton(module_dir)
+    if skel:
+        if skel.is_dir():
+            for f in sorted(skel.glob("*.json")):
+                try:
+                    h.update(f.read_bytes())
+                except OSError:
+                    pass
+        elif skel.is_file():
+            try:
+                h.update(skel.read_bytes())
+            except OSError:
+                pass
+
+    fl_dir = file_list_dir(module_dir)
+    if fl_dir.is_dir():
+        for f in sorted(fl_dir.iterdir()):
+            if f.is_file():
+                try:
+                    h.update(f.read_bytes())
+                except OSError:
+                    pass
+
+    return h.hexdigest()[:16]
+
+
+def get_cached_plan(module_dir: Path, doc_type: str, current_hash: str) -> dict | None:
+    """Get cached split plan if skeleton hash matches. Returns None on miss."""
+    cache = _load_cache(module_dir)
+    if cache.get("skeleton_hash") != current_hash:
+        return None
+    return cache.get("plans", {}).get(doc_type)
+
+
+def get_cached_split_count(module_dir: Path, doc_type: str, current_hash: str) -> int | None:
+    """Get previous split count. Returns None if no cache or hash changed."""
+    cache = _load_cache(module_dir)
+    if cache.get("skeleton_hash") != current_hash:
+        return None
+    return cache.get("split_counts", {}).get(doc_type)
+
+
+def save_plan_cache(module_dir: Path, skeleton_hash: str, doc_type: str, plan: dict, num_splits: int = 0) -> None:
+    """Save split plan to cache. Clears old cache if hash changed."""
+    from core.paths import meta_dir, ensure_dir
+
+    cache_path = meta_dir(module_dir) / _CACHE_FILE
+    ensure_dir(cache_path.parent)
+
+    cache = _load_cache(module_dir)
+    if cache.get("skeleton_hash") != skeleton_hash:
+        cache = {"skeleton_hash": skeleton_hash, "plans": {}, "split_counts": {}}
+
+    cache.setdefault("plans", {})[doc_type] = plan
+    if num_splits:
+        cache.setdefault("split_counts", {})[doc_type] = num_splits
+
+    try:
+        cache_path.write_text(json.dumps(cache, ensure_ascii=False, indent=2), encoding="utf-8")
+    except OSError as e:
+        logger.debug("Split cache write failed: %s", e)
+
+
+def _load_cache(module_dir: Path) -> dict:
+    """Load cache file. Returns empty dict on any error."""
+    from core.paths import meta_dir
+
+    cache_path = meta_dir(module_dir) / _CACHE_FILE
+    if not cache_path.exists():
+        return {}
+    try:
+        return json.loads(cache_path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return {}