source-kb 0.2.2__py3-none-any.whl

core/skeleton/dispatch_render.py

@@ -0,0 +1,319 @@
+"""Dispatch plan rendering — Markdown preview table and statistics formatting.
+
+Renders a DispatchPlan into human-readable Markdown tables for Gate 3A preview.
+No CLI I/O, no LLM calls.
+
+Usage:
+    from core.skeleton.dispatch import compute_dispatch_plan
+    from core.skeleton.dispatch_render import render_markdown, render_note
+
+    plan = compute_dispatch_plan(...)
+    markdown = render_markdown(plan)
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from core.skeleton.dispatch import DispatchEntry, DispatchPlan
+
+
+def write_shard_files(plan: DispatchPlan, module_dir: Path) -> int:
+    """Write shard file lists to .meta/shards/ for split entries.
+
+    Returns the number of shard files written.
+    """
+    count = 0
+    shards_dir = module_dir / ".meta" / "shards"
+    for entry in plan.entries:
+        if entry.split_count > 1 and entry.shards:
+            shards_dir.mkdir(parents=True, exist_ok=True)
+            for idx, shard in enumerate(entry.shards, 1):
+                shard_path = shards_dir / f"{entry.doc_type}-shard-{idx:02d}.txt"
+                file_paths = shard.files if shard.files else []
+                if file_paths:
+                    shard_path.write_text("\n".join(file_paths) + "\n", encoding="utf-8")
+                    count += 1
+    return count
+
+
+# ---------------------------------------------------------------------------
+# Note computation
+# ---------------------------------------------------------------------------
+
+
+def compute_note(
+    entry: DispatchEntry,
+    mode: str = "readwrite",
+    skeleton_size_kb: int = 0,
+) -> str:
+    """Generate meaningful note content for a dispatch entry.
+
+    Priority: risk warning > execution constraint > doc-type hint.
+    """
+    notes: list[str] = []
+    # Use mode-specific defaults (these match doc_types.yaml split config)
+    _DISPLAY_MAX_BYTES = {"readwrite": 2097152, "output-only": 307200}
+    max_bytes = _DISPLAY_MAX_BYTES.get(mode, 512000)
+
+    # 1. Risk: approaching byte limit
+    if entry.total_bytes > 0 and entry.doc_type not in ("source-tree-analysis", "index"):
+        usage_pct = entry.total_bytes / max_bytes
+        if usage_pct > 0.9:
+            notes.append(f"⚠️ Approaching limit ({entry.total_bytes // 1024}KB/{max_bytes // 1024}KB)")
+        elif usage_pct > 0.75:
+            notes.append(f"Large input ({int(usage_pct * 100)}%)")
+
+    # 2. Doc-type specific constraints
+    if entry.doc_type == "architecture":
+        notes.append("Must not produce redis/kafka/business-logic content")
+    elif entry.doc_type == "index":
+        notes.append("Depends on all prior tasks completing")
+    elif entry.doc_type == "source-tree-analysis" and skeleton_size_kb > 50:
+        notes.append("Use skeleton-summary (skeleton too large)")
+    elif entry.doc_type == "business-logic":
+        notes.append("Organize by business flow, not by class name")
+
+    # 3. Large file count warning
+    if entry.file_count > 80 and entry.doc_type not in ("source-tree-analysis", "index"):
+        notes.append(f"{entry.file_count} files, requires batched reading")
+
+    return " | ".join(notes) if notes else ""
+
+
+# ---------------------------------------------------------------------------
+# Markdown rendering
+# ---------------------------------------------------------------------------
+
+
+def render_markdown(plan: DispatchPlan, mode: str = "readwrite") -> str:
+    """Render a DispatchPlan as a Markdown document.
+
+    Args:
+        plan: Computed dispatch plan
+        mode: Execution mode for note computation
+
+    Returns:
+        Formatted Markdown string with header, table, shard details, and summary
+    """
+    lines: list[str] = []
+
+    # Title
+    lines.append(f"# Dispatch Plan: {plan.module_name}")
+    lines.append("")
+    lines.append(f"Module: {plan.module_name} ({plan.source_file_count} source files, "
+                 f"{plan.source_total_lines} lines)")
+    lines.append(f"Skeleton size: {plan.skeleton_size_kb}KB")
+    lines.append(f"Mode: {'Agent (readwrite)' if mode == 'readwrite' else 'CLI (output-only)'}")
+    lines.append("")
+
+    # Dispatch table
+    lines.append("## Dispatch Plan")
+    lines.append("")
+    lines.append("| Batch | Sub-agent | Output Doc | Input Files | Input Lines | Input Bytes | Split Strategy | Notes |")
+    lines.append("|------|--------|---------|-----------|-----------|-----------|---------|------|")
+
+    for entry in plan.entries:
+        note = compute_note(entry, mode=mode, skeleton_size_kb=plan.skeleton_size_kb)
+        row = _format_row(entry, note)
+        lines.append(row)
+
+    # Shard details (for entries with split_count > 1)
+    sharded = [e for e in plan.entries if e.split_count > 1]
+    if sharded:
+        lines.append("")
+        lines.append("## Shard Details")
+        lines.append("")
+        for entry in sharded:
+            lines.append(f"### {entry.doc_type} (split into {entry.split_count})")
+            lines.append("")
+            if entry.pending_grouping:
+                lines.append(f"> **Pending Agent grouping**: Read the file list in `{entry.pending_grouping}`,")
+                lines.append(f"> split into {entry.split_count} groups by business domain, then run `split-apply` to backfill.")
+                lines.append("")
+            lines.append("| Shard | Domain | Files |")
+            lines.append("|------|--------|--------|")
+            if entry.shards:
+                for i, shard in enumerate(entry.shards, 1):
+                    name = shard.name or f"group-{i}"
+                    lines.append(f"| shard-{i:02d} | {name} | {shard.file_count} |")
+            else:
+                per_shard = entry.file_count // entry.split_count if entry.file_count else 0
+                for i in range(1, entry.split_count + 1):
+                    lines.append(f"| shard-{i:02d} | (pending Agent grouping) | ~{per_shard} |")
+            lines.append("")
+
+    # Summary
+    lines.append("## Summary")
+    lines.append("")
+    lines.append(f"- Sub-agent tasks: {plan.total_shards()}")
+    lines.append(f"- Total input files: {plan.source_file_count}")
+    lines.append(f"- Total input lines: {plan.source_total_lines}")
+
+    return "\n".join(lines)
+
+
+def render_summary(plan: DispatchPlan) -> str:
+    """Render a brief summary of the dispatch plan.
+
+    Returns:
+        Short summary string (e.g., "12 tasks across 4 batches, 3 need splitting")
+    """
+    total = len(plan.entries)
+    batches = len(set(e.batch_id for e in plan.entries))
+    splits = sum(1 for e in plan.entries if e.split_count > 1)
+
+    parts = [f"{total} tasks", f"{batches} batches"]
+    if splits:
+        parts.append(f"{splits} need splitting")
+    return ", ".join(parts)
+
+
+def render_batch_groups(plan: DispatchPlan) -> list[tuple[str, list[DispatchEntry]]]:
+    """Group entries by batch for sequential execution.
+
+    Returns:
+        List of (batch_id, entries) tuples in batch order
+    """
+    groups: dict[str, list[DispatchEntry]] = {}
+    for entry in plan.entries:
+        groups.setdefault(entry.batch_id, []).append(entry)
+
+    # Maintain batch_order ordering
+    result: list[tuple[str, list[DispatchEntry]]] = []
+    seen: set[str] = set()
+    for batch_id, _ in plan.batch_order:
+        if batch_id in groups and batch_id not in seen:
+            result.append((batch_id, groups[batch_id]))
+            seen.add(batch_id)
+
+    # Append any remaining
+    for batch_id, entries in groups.items():
+        if batch_id not in seen:
+            result.append((batch_id, entries))
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Task manifest generation (shared by CLI and Agent modes)
+# ---------------------------------------------------------------------------
+
+
+def plan_to_tasks(
+    plan: DispatchPlan,
+    kb_name: str,
+    preset_name: str,
+    preset: dict,
+    knowledge_dir,
+    mode: str = "readwrite",
+) -> dict:
+    """Convert DispatchPlan to task manifest JSON.
+
+    Shared by CLI DispatchPlanStep and Agent dispatch-preview command.
+    """
+    from pathlib import Path
+    from core.preset import get_template_path
+
+    module_name = plan.module_name
+    module_dir = Path(knowledge_dir) / module_name
+
+    tasks: list[dict] = []
+    for entry in plan.entries:
+        dt = entry.doc_type
+        template_name = get_template_path(preset, dt, preset_name) or ""
+
+        if entry.split_count <= 1:
+            output_file = str((module_dir / entry.doc_filename).resolve())
+            render_cmd = (
+                f"py -3.12 -X utf8 -m core.prompt render"
+                f" --template \"{template_name}\""
+                f" --module {module_name}"
+                f" --config \"kb-project.yaml\""
+                f" --kb {kb_name}"
+                f" --doc-type {dt}"
+                f" --mode {mode}"
+            )
+            tasks.append({
+                "id": dt,
+                "batch": entry.batch_id,
+                "doc_type": dt,
+                "doc_filename": entry.doc_filename,
+                "shard": None,
+                "total_shards": 1,
+                "file_count": entry.file_count,
+                "line_count": entry.total_lines,
+                "output_file": output_file,
+                "render_cmd": render_cmd,
+                "template": template_name,
+            })
+        else:
+            for shard_idx in range(1, entry.split_count + 1):
+                shard_filename = f"{entry.doc_filename.replace('.md', '')}-shard-{shard_idx:02d}.md"
+                output_file = str((module_dir / shard_filename).resolve())
+                fl_override = str(
+                    (module_dir / ".meta" / "shards" / f"{dt}-shard-{shard_idx:02d}.txt").resolve()
+                )
+                render_cmd = (
+                    f"py -3.12 -X utf8 -m core.prompt render"
+                    f" --template \"{template_name}\""
+                    f" --module {module_name}"
+                    f" --config \"kb-project.yaml\""
+                    f" --kb {kb_name}"
+                    f" --doc-type {dt}"
+                    f" --mode {mode}"
+                    f" --extra file_list_override={fl_override}"
+                )
+                # Use actual shard data when available
+                if entry.shards and shard_idx <= len(entry.shards):
+                    shard_info = entry.shards[shard_idx - 1]
+                    shard_file_count = shard_info.file_count
+                    shard_line_count = shard_info.lines
+                else:
+                    shard_file_count = entry.file_count // entry.split_count
+                    shard_line_count = entry.total_lines // entry.split_count
+                tasks.append({
+                    "id": f"{dt}-shard-{shard_idx:02d}",
+                    "batch": entry.batch_id,
+                    "doc_type": dt,
+                    "doc_filename": entry.doc_filename,
+                    "shard": shard_idx,
+                    "total_shards": entry.split_count,
+                    "file_count": shard_file_count,
+                    "line_count": shard_line_count,
+                    "output_file": output_file,
+                    "render_cmd": render_cmd,
+                    "template": template_name,
+                    "shard_file_list": fl_override,
+                })
+
+    return {
+        "module": module_name,
+        "kb": kb_name,
+        "mode": mode,
+        "preset": preset_name,
+        "tasks": tasks,
+    }
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _format_row(entry: DispatchEntry, note: str) -> str:
+    """Format a single table row."""
+    fc = str(entry.file_count) if entry.file_count else "—"
+    lc = str(entry.total_lines) if entry.total_lines else "—"
+    bc = f"{entry.total_bytes // 1024}KB" if entry.total_bytes > 0 else "—"
+
+    return (
+        f"| {entry.batch_id:<4} "
+        f"| {entry.doc_type:<14} "
+        f"| {entry.doc_filename:<28} "
+        f"| {fc:<9} "
+        f"| {lc:<9} "
+        f"| {bc:<9} "
+        f"| {entry.strategy:<7} "
+        f"| {note} |"
+    )

core/skeleton/dispatch_source.py

@@ -0,0 +1,111 @@
+"""Dispatch source helpers — file list resolution and source statistics."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any
+
+
+def get_file_list_with_stats(
+    module_dir: Path, doc_type: str, source_cache: Path, dt_mapping: dict[str, str],
+) -> list[dict[str, Any]]:
+    """Get file list with path, lines, and bytes info for a doc type."""
+    from core.paths import resolve_file_list
+    if doc_type not in dt_mapping:
+        return []
+    fl = resolve_file_list(module_dir, doc_type)
+    if fl is None:
+        doc_basename = dt_mapping[doc_type].replace(".md", "")
+        if doc_basename != doc_type:
+            fl = resolve_file_list(module_dir, doc_basename)
+    if fl is None:
+        return []
+    files: list[dict[str, Any]] = []
+    for line in fl.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        rel_path = line.replace("\\", "/")
+        file_path = source_cache / rel_path
+        if not file_path.exists() and Path(line).is_absolute():
+            file_path = Path(line)
+        file_bytes, line_count = 0, 0
+        if file_path.exists():
+            try:
+                file_bytes = file_path.stat().st_size
+            except OSError:
+                pass
+            try:
+                with open(file_path, encoding="utf-8", errors="replace") as f:
+                    line_count = sum(1 for _ in f)
+            except OSError:
+                pass
+        package = _extract_package_from_path(rel_path)
+        files.append({"path": str(file_path), "name": Path(rel_path).name,
+                      "lines": line_count, "bytes": file_bytes, "rel_path": rel_path,
+                      "package": package})
+    return files
+
+
+def _extract_package_from_path(rel_path: str) -> str:
+    """Extract package name from a relative file path.
+
+    Skips standard source directory prefixes (src/main/java/, etc.)
+    and joins remaining directory parts with '.' as package name.
+    """
+    parts = rel_path.replace("\\", "/").split("/")
+    skip_dirs = {"src", "main", "java", "python", "go", "kotlin", "scala",
+                 "groovy", "lib", "app", "pkg", "internal", "cmd", "resources", "test"}
+
+    start = 0
+    i = 0
+    while i < len(parts) - 1:
+        if parts[i].lower() in skip_dirs:
+            j = i
+            while j < len(parts) - 1 and parts[j].lower() in skip_dirs:
+                j += 1
+            start = j
+            i = j
+        else:
+            i += 1
+    segments = [p for p in parts[start:-1] if p]
+    return ".".join(segments)
+
+
+def count_source_lines(source_cache: Path, hooks=None) -> tuple[int, int]:
+    """Return (source_file_count, total_lines) for source files."""
+    if hooks:
+        result = hooks.count_source_files(source_cache)
+        if result != (0, 0):
+            return result
+
+    total_files, total_lines = 0, 0
+    if not source_cache.is_dir():
+        return 0, 0
+    extensions = hooks.get_source_extensions() if hooks else [".java"]
+    test_patterns = hooks.get_test_path_patterns() if hooks else ["/test/"]
+    for root, _, filenames in os.walk(source_cache):
+        root_str = str(Path(root)).replace("\\", "/")
+        if any(pat in root_str for pat in test_patterns):
+            continue
+        for fn in filenames:
+            if any(fn.endswith(ext) for ext in extensions):
+                total_files += 1
+                try:
+                    with open(Path(root) / fn, encoding="utf-8", errors="replace") as f:
+                        total_lines += sum(1 for _ in f)
+                except OSError:
+                    pass
+    return total_files, total_lines
+
+
+def skeleton_total_size_kb(module_dir: Path) -> int:
+    """Get total skeleton size in KB."""
+    from core.paths import resolve_skeleton
+    skel = resolve_skeleton(module_dir)
+    if skel is None:
+        return 0
+    if skel.is_dir():
+        return sum(f.stat().st_size for f in skel.glob("*.json")) // 1024
+    return skel.stat().st_size // 1024

core/skeleton/extract.py

@@ -0,0 +1,218 @@
+"""Skeleton extraction orchestrator.
+
+Tries parsers in priority order (from preset config) until one succeeds.
+Handles output modes: single file, split-by-package, summary generation.
+
+Usage:
+    from core.skeleton.extract import extract_skeleton
+
+    entries = extract_skeleton(repo_path, preset, ref="origin/main")
+    # entries: list of skeleton dicts [{file, total_lines, methods, classes, ...}]
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+from core.paths import (
+    skeleton_path, skeleton_shards_dir, skeleton_summary_path, skeleton_stats_path, ensure_dir,
+)
+from core.skeleton.parsers import get_parser_chain
+
+logger = logging.getLogger(__name__)
+
+
+def extract_skeleton(
+    repo_path: Path,
+    preset: dict[str, Any],
+    *,
+    ref: str = "HEAD",
+    subpath: str | None = None,
+    output_dir: Path | None = None,
+    split_by_package: bool = False,
+    compact: bool = True,
+    files: list[str] | None = None,
+) -> list[dict[str, Any]]:
+    """Extract skeleton from a repository using the parser chain.
+
+    Tries each parser in preset-declared order until one succeeds.
+
+    Args:
+        repo_path: Path to the git repository
+        preset: Preset configuration dict
+        ref: Git reference to parse
+        subpath: Subdirectory to scope extraction (monorepo module path)
+        output_dir: Module dir to write output (None = don't write)
+        split_by_package: Split output into per-package shard files
+        compact: Remove low-value entries (empty methods, etc.)
+        files: If provided, only extract these file paths (incremental mode)
+
+    Returns:
+        List of skeleton entries
+    """
+    chain = get_parser_chain(preset)
+    entries: list[dict[str, Any]] = []
+    used_parser = "none"
+
+    for parser in chain:
+        if not parser.can_parse(repo_path, preset):
+            logger.debug("Parser '%s' cannot parse %s, skipping", parser.name, repo_path)
+            continue
+
+        logger.info("Using parser '%s' for %s", parser.name, repo_path.name)
+        try:
+            entries = parser.parse(repo_path, preset, ref=ref, subpath=subpath)
+            used_parser = parser.name
+            break
+        except Exception as e:
+            logger.warning("Parser '%s' failed for %s: %s", parser.name, repo_path.name, e)
+            continue
+
+    if not entries:
+        logger.warning("No parser produced results for %s", repo_path.name)
+        return []
+
+    # Filter to specified files if in incremental mode
+    if files:
+        file_set = set(files)
+        before_count = len(entries)
+        entries = [e for e in entries if e.get("file", "") in file_set]
+        skipped = file_set - {e.get("file", "") for e in entries}
+        if skipped:
+            logger.warning("[extract] %d files not found in skeleton: %s",
+                          len(skipped), list(skipped)[:5])
+        logger.info("[extract] Incremental: %d/%d entries (from %d requested files)",
+                    len(entries), before_count, len(files))
+
+    if compact:
+        entries = _compact(entries)
+
+    # Write output if requested
+    if output_dir:
+        _write_output(entries, output_dir, split_by_package)
+        _write_summary(entries, output_dir)
+        _write_stats(entries, output_dir)
+        _write_parser_info(output_dir, used_parser, preset.get("preset_name", ""))
+
+    logger.info("Extracted %d entries (%d methods) using '%s'",
+                len(entries), sum(len(e.get("methods", [])) for e in entries), used_parser)
+    return entries
+
+
+def _compact(entries: list[dict]) -> list[dict]:
+    """Remove entries with no methods and no classes (low value)."""
+    return [e for e in entries if e.get("methods") or e.get("classes")]
+
+
+def _write_output(entries: list[dict], module_dir: Path, split_by_package: bool) -> None:
+    """Write skeleton entries to disk."""
+    if split_by_package and len(entries) > 50:
+        _write_shards(entries, module_dir)
+    else:
+        _write_single(entries, module_dir)
+
+
+def _write_single(entries: list[dict], module_dir: Path) -> None:
+    """Write all entries to a single skeleton.json file."""
+    out = skeleton_path(module_dir)
+    ensure_dir(out.parent)
+    out.write_text(json.dumps(entries, ensure_ascii=False, indent=1), encoding="utf-8")
+    logger.debug("Wrote skeleton: %s (%d entries)", out, len(entries))
+
+
+def _write_shards(entries: list[dict], module_dir: Path) -> None:
+    """Split entries by package prefix and write per-package shard files."""
+    shards_dir = skeleton_shards_dir(module_dir)
+    ensure_dir(shards_dir)
+
+    # Group by top-level package
+    groups: dict[str, list[dict]] = {}
+    for entry in entries:
+        filepath = entry.get("file", "")
+        pkg = _extract_package(filepath)
+        groups.setdefault(pkg, []).append(entry)
+
+    for pkg_name, pkg_entries in groups.items():
+        safe_name = pkg_name.replace(".", "_").replace("/", "_") or "root"
+        shard_file = shards_dir / f"{safe_name}.json"
+        shard_file.write_text(
+            json.dumps(pkg_entries, ensure_ascii=False, indent=1), encoding="utf-8"
+        )
+
+    logger.debug("Wrote %d skeleton shards to %s", len(groups), shards_dir)
+
+
+def _write_summary(entries: list[dict], module_dir: Path) -> None:
+    """Write a compact summary (file + method count + high methods) for large skeletons."""
+    summary = []
+    for entry in entries:
+        methods = entry.get("methods", [])
+        high_methods = [m for m in methods if m.get("complexity") == "high"]
+        summary.append({
+            "file": entry.get("file", ""),
+            "total_lines": entry.get("total_lines", 0),
+            "method_count": len(methods),
+            "high_complexity_methods": [
+                {"name": m["name"], "line_count": m.get("line_count", 0)}
+                for m in high_methods
+            ],
+        })
+
+    out = skeleton_summary_path(module_dir)
+    ensure_dir(out.parent)
+    out.write_text(json.dumps(summary, ensure_ascii=False, indent=1), encoding="utf-8")
+
+
+def _write_stats(entries: list[dict], module_dir: Path) -> None:
+    """Write global statistics for downstream consumers."""
+    total_methods = sum(len(e.get("methods", [])) for e in entries)
+    total_classes = sum(len(e.get("classes", [])) for e in entries)
+    total_fields = sum(len(e.get("fields", [])) for e in entries)
+    total_lines = sum(e.get("total_lines", 0) for e in entries)
+    high_complexity = sum(
+        1 for e in entries
+        for m in e.get("methods", [])
+        if m.get("complexity") == "high"
+    )
+
+    stats = {
+        "files": len(entries),
+        "methods": total_methods,
+        "classes": total_classes,
+        "fields": total_fields,
+        "total_lines": total_lines,
+        "high_complexity_methods": high_complexity,
+    }
+
+    out = skeleton_stats_path(module_dir)
+    ensure_dir(out.parent)
+    out.write_text(json.dumps(stats, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def _write_parser_info(module_dir: Path, parser_name: str, preset_name: str) -> None:
+    """Write parser metadata for audit trail."""
+    from core.paths import meta_dir
+    info_path = meta_dir(module_dir) / "parser-info.json"
+    ensure_dir(info_path.parent)
+    info_path.write_text(
+        json.dumps({"parser": parser_name, "preset": preset_name}, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )
+
+
+def _extract_package(filepath: str, depth: int = 2) -> str:
+    """Extract top-level package from file path for sharding."""
+    parts = filepath.replace("\\", "/").split("/")
+    # Skip standard source prefixes
+    skip = {"src", "main", "java", "python", "go", "kotlin", "lib", "app", "pkg"}
+    start = 0
+    for i, p in enumerate(parts[:-1]):
+        if p.lower() in skip:
+            start = i + 1
+        else:
+            break
+    remaining = parts[start:-1]
+    return ".".join(remaining[:depth]) if remaining else "root"