source-kb 0.2.2__py3-none-any.whl

core/skeleton/split_plan_llm.py

@@ -0,0 +1,274 @@
+"""Split plan LLM strategy — LLM-assisted business-domain grouping.
+
+Handles both CLI mode (direct LLM call) and delegated mode (Agent dispatch).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+from core.skeleton.split import SplitConfig, SplitPlan
+from core.skeleton.split_plan_helpers import (
+    make_split, derive_name, rebalance_groups,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def try_llm_split(
+    file_list: list[dict],
+    split_config: SplitConfig,
+    doc_type: str,
+    module_dir: Path | None,
+    *,
+    strategy: Any | None = None,
+    is_delegated: bool = False,
+) -> SplitPlan | None:
+    """Try LLM-assisted business-domain grouping.
+
+    In Agent mode (delegated): outputs a grouping request JSON file for the Agent
+    to process using its own LLM capability, then returns a pending plan.
+
+    In CLI mode: calls the provided LLM strategy directly.
+
+    Args:
+        file_list: Files to group.
+        split_config: Split configuration.
+        doc_type: Document type being split.
+        module_dir: Module directory for cache/shard files.
+        strategy: Injected LlmStrategy instance (from engine/ caller). None = skip LLM.
+        is_delegated: Whether running in delegated mode (Agent handles grouping).
+    """
+    if not module_dir:
+        return None
+
+    from core.skeleton.split import compute_splits
+
+    max_files = split_config.max_files_per_shard
+    total_lines = sum(f.get("lines", 0) for f in file_list)
+    total_bytes = sum(f.get("bytes", 0) for f in file_list)
+    n_target = max(2, compute_splits(
+        split_config,
+        total_bytes=total_bytes,
+        total_lines=total_lines,
+        file_count=len(file_list),
+        doc_type=doc_type,
+    ))
+    max_lines = split_config.effective_max_lines(doc_type)
+
+    if is_delegated:
+        existing = _load_existing_shards(module_dir, doc_type, file_list)
+        if existing:
+            return existing
+        return _emit_grouping_request(file_list, module_dir, doc_type, n_target, max_files, max_lines)
+
+    # CLI mode: need a strategy to call LLM
+    if strategy is None:
+        return None
+
+    llm_sample_limit = split_config.llm_sample_limit
+    sampled_files = file_list if len(file_list) <= llm_sample_limit else file_list[:llm_sample_limit]
+
+    file_summaries = []
+    for f in sampled_files:
+        name = f.get("name", "")
+        pkg = f.get("package", "")
+        lines = f.get("lines", 0)
+        file_summaries.append(f"{pkg}.{name} ({lines} lines)")
+
+    count_desc = f"{len(sampled_files)} Java files"
+    if len(sampled_files) < len(file_list):
+        count_desc += f" (sample of {len(file_list)} total)"
+    prompt_user = (
+        f"Group the following {count_desc} into {n_target} groups by business domain.\n"
+        f"Output format per group: `GroupName: File1, File2, ...`\n"
+        f"File list:\n" + "\n".join(file_summaries)
+    )
+
+    from core.interfaces import LlmRequest
+    try:
+        resp = strategy.call(LlmRequest(
+            system=(
+                "You are a Java project architect skilled at grouping source files by business domain. "
+                "Output only the grouping result, no explanation."
+            ),
+            user=prompt_user,
+            max_tokens=2000,
+            temperature=0.1,
+        ))
+    except Exception:
+        return None
+
+    if resp.status != "done" or not resp.content:
+        return None
+
+    groups = _parse_llm_groups(resp.content, file_list)
+    if not groups or len(groups) <= 1:
+        return None
+
+    assigned = {id(f) for g in groups for f in g}
+    unassigned = [f for f in file_list if id(f) not in assigned]
+    if unassigned:
+        _assign_by_package(unassigned, groups)
+
+    rebalance_groups(groups, split_config.max_files_per_shard)
+
+    splits = [make_split(derive_name(g, noise_words=split_config.noise_words), g) for g in groups]
+    return SplitPlan(splits=splits, strategy="llm")
+
+
+def _load_existing_shards(
+    module_dir: Path,
+    doc_type: str,
+    file_list: list[dict],
+) -> SplitPlan | None:
+    """Load existing shard files produced by a prior split-apply."""
+    shards_dir = module_dir / ".meta" / "shards"
+    if not shards_dir.is_dir():
+        return None
+
+    shard_files = sorted(shards_dir.glob(f"{doc_type}-shard-*.txt"))
+    if not shard_files:
+        return None
+
+    name_to_file = {f.get("name", ""): f for f in file_list}
+
+    splits: list[dict[str, Any]] = []
+    group_names: list[str] = []
+    groups_path = module_dir / ".meta" / "split-requests" / f"{doc_type}-groups.json"
+    if groups_path.exists():
+        try:
+            groups_data = json.loads(groups_path.read_text(encoding="utf-8"))
+            group_names = [g.get("name", "") for g in groups_data]
+        except Exception:
+            pass
+
+    for idx, shard_file in enumerate(shard_files):
+        filenames = [
+            ln.strip() for ln in shard_file.read_text(encoding="utf-8").splitlines()
+            if ln.strip()
+        ]
+        shard_files_info = [name_to_file[fn] for fn in filenames if fn in name_to_file]
+        name = group_names[idx] if idx < len(group_names) else f"group-{idx + 1}"
+        splits.append(make_split(name, shard_files_info))
+
+    if not splits or len(splits) <= 1:
+        return None
+
+    logger.info("split strategy=existing-shards doc_type=%s splits=%d", doc_type, len(splits))
+    return SplitPlan(splits=splits, strategy="agent-applied")
+
+
+def _emit_grouping_request(
+    file_list: list[dict],
+    module_dir: Path,
+    doc_type: str,
+    n_target: int,
+    max_files: int,
+    max_lines: int,
+) -> SplitPlan:
+    """Write a grouping request JSON for Agent to process."""
+    request_dir = module_dir / ".meta" / "split-requests"
+    request_dir.mkdir(parents=True, exist_ok=True)
+    request_path = request_dir / f"{doc_type}-grouping-request.json"
+
+    file_entries = []
+    for f in file_list:
+        file_entries.append({
+            "name": f.get("name", ""),
+            "package": f.get("package", ""),
+            "lines": f.get("lines", 0),
+            "bytes": f.get("bytes", 0),
+        })
+
+    request = {
+        "doc_type": doc_type,
+        "n_target": n_target,
+        "constraints": {
+            "max_files_per_group": min(max_files, 80),
+            "max_lines_per_group": max_lines,
+            "max_imbalance_ratio": 3.0,
+            "all_files_must_be_assigned": True,
+            "no_duplicate_assignment": True,
+        },
+        "files": file_entries,
+        "output_format": {
+            "description": "JSON array of groups",
+            "example": [
+                {"name": "order-lifecycle", "files": ["OrderServiceImpl.java", "OrderProcessor.java"]},
+                {"name": "payment", "files": ["PaymentServiceImpl.java"]},
+            ],
+        },
+    }
+
+    request_path.write_text(json.dumps(request, ensure_ascii=False, indent=2), encoding="utf-8")
+    logger.info("Agent grouping request written to %s", request_path)
+
+    return SplitPlan(
+        splits=[],
+        strategy="agent-pending",
+        pending_agent_grouping=True,
+        grouping_request_path=str(request_path),
+    )
+
+
+def _parse_llm_groups(content: str, file_list: list[dict]) -> list[list[dict]] | None:
+    """Parse LLM grouping response into file groups."""
+    name_to_file = {f.get("name", ""): f for f in file_list}
+    groups: list[list[dict]] = []
+
+    for line in content.strip().splitlines():
+        if ":" not in line:
+            continue
+        _, files_str = line.split(":", 1)
+        group: list[dict] = []
+        for fname in files_str.split(","):
+            fname = fname.strip().split("(")[0].strip()
+            if fname in name_to_file:
+                group.append(name_to_file[fname])
+            else:
+                class_name = fname.rsplit(".", 1)[-1] if "." in fname else fname
+                for name, f in name_to_file.items():
+                    if name.endswith(class_name + ".java") or name == class_name:
+                        group.append(f)
+                        break
+        if group:
+            groups.append(group)
+
+    return groups if len(groups) > 1 else None
+
+
+def _assign_by_package(unassigned: list[dict], groups: list[list[dict]]) -> None:
+    """Assign unassigned files to the group with the most package overlap."""
+    group_packages: list[set[str]] = []
+    for g in groups:
+        pkgs = set()
+        for f in g:
+            pkg = f.get("package", "")
+            while pkg:
+                pkgs.add(pkg)
+                pkg = pkg.rsplit(".", 1)[0] if "." in pkg else ""
+        group_packages.append(pkgs)
+
+    for f in unassigned:
+        pkg = f.get("package", "")
+        best_idx = 0
+        best_score = -1
+        for i, pkgs in enumerate(group_packages):
+            score = 0
+            p = pkg
+            while p:
+                if p in pkgs:
+                    score += 1
+                p = p.rsplit(".", 1)[0] if "." in p else ""
+            if score > best_score:
+                best_score = score
+                best_idx = i
+        groups[best_idx].append(f)
+        pkg_full = f.get("package", "")
+        while pkg_full:
+            group_packages[best_idx].add(pkg_full)
+            pkg_full = pkg_full.rsplit(".", 1)[0] if "." in pkg_full else ""

core/utils.py

@@ -0,0 +1,135 @@
+"""Shared utility functions — text sanitization, subprocess safety, path validation.
+
+Pure utility functions with no LLM calls or CLI I/O.
+
+Usage:
+    from core.utils import sanitize_llm_input, safe_subprocess_run, validate_path_within_bounds
+"""
+
+from __future__ import annotations
+
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+
+# ---------------------------------------------------------------------------
+# Path validation
+# ---------------------------------------------------------------------------
+
+
+def validate_path_within_bounds(path: Path, base: Path) -> bool:
+    """Security check: ensure resolved path doesn't escape base directory.
+
+    Args:
+        path: Path to validate (may be relative or absolute).
+        base: Base directory that path must stay within.
+
+    Returns:
+        True if path is within bounds, False otherwise.
+    """
+    try:
+        resolved = path.resolve()
+        base_resolved = base.resolve()
+    except (OSError, ValueError):
+        return False
+
+    if sys.platform == "win32":
+        return str(resolved).lower().startswith(str(base_resolved).lower())
+    else:
+        try:
+            resolved.relative_to(base_resolved)
+            return True
+        except ValueError:
+            return False
+
+
+# ---------------------------------------------------------------------------
+# Text sanitization for LLM input
+# ---------------------------------------------------------------------------
+
+_INJECTION_PATTERNS = [
+    re.compile(r"(?i)ignore\s+(all\s+)?previous\s+instructions"),
+    re.compile(r"(?i)^system\s*:", re.MULTILINE),
+    re.compile(r"(?i)you\s+are\s+now\s+"),
+]
+
+MAX_LLM_INPUT_CHARS = 200_000
+
+
+def sanitize_llm_input(text: str) -> str:
+    """Clean text for LLM input: remove null bytes, normalize whitespace, filter injections.
+
+    - Removes null bytes
+    - Truncates to MAX_LLM_INPUT_CHARS
+    - Filters common prompt injection patterns
+
+    Args:
+        text: Raw text to sanitize.
+
+    Returns:
+        Sanitized text safe for LLM consumption.
+    """
+    # Remove null bytes
+    text = text.replace("\0", "")
+
+    # Truncate
+    if len(text) > MAX_LLM_INPUT_CHARS:
+        text = text[:MAX_LLM_INPUT_CHARS] + "\n[truncated]"
+
+    # Filter injection patterns
+    for pat in _INJECTION_PATTERNS:
+        text = pat.sub("[filtered]", text)
+
+    return text
+
+
+# ---------------------------------------------------------------------------
+# Safe subprocess execution
+# ---------------------------------------------------------------------------
+
+
+def safe_subprocess_run(
+    cmd: list[str | Path],
+    *,
+    timeout: int = 120,
+    capture: bool = True,
+    env: dict[str, str] | None = None,
+    cwd: Path | str | None = None,
+) -> subprocess.CompletedProcess:
+    """Subprocess wrapper with timeout and encoding handling.
+
+    All Path arguments are resolved to absolute paths to prevent injection.
+    Uses UTF-8 encoding with error replacement.
+
+    Args:
+        cmd: Command and arguments (Path objects are resolved to absolute).
+        timeout: Maximum execution time in seconds.
+        capture: Whether to capture stdout/stderr.
+        env: Optional environment variables override.
+        cwd: Optional working directory.
+
+    Returns:
+        CompletedProcess result.
+
+    Raises:
+        subprocess.TimeoutExpired: If command exceeds timeout.
+        FileNotFoundError: If command executable not found.
+    """
+    sanitized: list[str] = []
+    for arg in cmd:
+        if isinstance(arg, Path):
+            sanitized.append(str(arg.resolve()))
+        else:
+            sanitized.append(str(arg))
+
+    return subprocess.run(
+        sanitized,
+        capture_output=capture,
+        encoding="utf-8",
+        errors="replace",
+        timeout=timeout,
+        env=env,
+        cwd=str(cwd) if cwd else None,
+    )

core/validators/__init__.py

@@ -0,0 +1,65 @@
+"""core.validators — document quality validators (registry pattern).
+
+Each validator implements the Validator ABC and is registered by name.
+Validators are pure functions: they inspect files and return structured results
+without performing any remediation.
+
+Usage:
+    from core.validators import get_all_validators, run_all
+
+    results = run_all(module_dir, preset_name="java-spring")
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from core.interfaces import Validator, ValidationResult
+
+_REGISTRY: dict[str, type[Validator]] = {}
+
+
+def register_validator(cls: type[Validator]) -> type[Validator]:
+    """Decorator to register a validator class."""
+    _REGISTRY[cls.name] = cls
+    return cls
+
+
+def get_all_validators() -> list[Validator]:
+    """Return instances of all registered validators."""
+    _ensure_loaded()
+    return [cls() for cls in _REGISTRY.values()]
+
+
+def get_validator(name: str) -> Validator:
+    """Get a specific validator by name."""
+    if name not in _REGISTRY:
+        raise KeyError(f"Validator '{name}' not found. Available: {sorted(_REGISTRY.keys())}")
+    return _REGISTRY[name]()
+
+
+def run_all(module_dir: Path, **kwargs: Any) -> ValidationResult:
+    """Run all registered validators and merge results."""
+    _ensure_loaded()
+    combined = ValidationResult()
+    for validator in get_all_validators():
+        try:
+            result = validator.validate(module_dir, **kwargs)
+            combined = combined.merge(result)
+        except Exception as e:
+            combined.errors.append(f"{validator.name}: {e}")
+    return combined
+
+
+def _ensure_loaded() -> None:
+    """Import all validator modules to trigger registration."""
+    if _REGISTRY:
+        return
+    # Import all validator modules so @register_validator decorators fire
+    from core.validators import duplicates  # noqa: F401
+    from core.validators import links  # noqa: F401
+    from core.validators import sampling  # noqa: F401
+    from core.validators import structure  # noqa: F401
+    from core.validators import coverage  # noqa: F401
+    from core.validators import consistency  # noqa: F401

core/validators/__main__.py

@@ -0,0 +1,215 @@
+"""CLI entry points for core/validators tools.
+
+Provides sub-commands for Agent mode to call directly:
+    python -m core.validators coverage ...
+    python -m core.validators consistency ...
+    python -m core.validators check-progress ...
+    python -m core.validators links ...
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+sys.stdout.reconfigure(encoding="utf-8")
+
+
+def cmd_coverage(args: argparse.Namespace) -> None:
+    """Run coverage check."""
+    from core.validators.coverage import CoverageValidator
+
+    module_dir = Path(args.module_dir) if args.module_dir else None
+    skeleton_path = Path(args.skeleton) if args.skeleton else None
+    skeleton_dir = Path(args.skeleton_dir) if args.skeleton_dir else None
+    docs_dir = Path(args.docs_dir) if args.docs_dir else module_dir
+
+    if not docs_dir:
+        print("Error: --docs-dir or --module-dir required", file=sys.stderr)
+        sys.exit(1)
+
+    validator = CoverageValidator()
+    result = validator.validate(
+        docs_dir,
+        skeleton_path=str(skeleton_path) if skeleton_path else None,
+        skeleton_dir=str(skeleton_dir) if skeleton_dir else None,
+        module_type=args.type or "service",
+    )
+
+    if result.errors:
+        print("ERRORS:")
+        for e in result.errors:
+            print(f"  ❌ {e}")
+    if result.warnings:
+        print("WARNINGS:")
+        for w in result.warnings:
+            print(f"  ⚠️ {w}")
+
+    status = "ok" if result.passed else "fail"
+    print(json.dumps({"status": status, "errors": len(result.errors),
+                      "warnings": len(result.warnings)}, ensure_ascii=False))
+
+
+def cmd_consistency(args: argparse.Namespace) -> None:
+    """Run consistency check (progress monitoring + reference validation)."""
+    from core.validators.consistency import ConsistencyValidator
+    from core.monitor.progress import check_progress, cleanup_progress
+
+    module_dir = Path(args.module_dir)
+
+    if args.cleanup:
+        cleaned = cleanup_progress(module_dir)
+        print(f"Cleaned {cleaned} progress files")
+        print(json.dumps({"status": "ok", "cleaned": cleaned}, ensure_ascii=False))
+        return
+
+    # Progress check
+    progress = check_progress(module_dir)
+    if progress:
+        print("Progress status:")
+        for doc_name, status in progress.items():
+            print(f"  {doc_name}: {status}")
+
+    # Consistency validation
+    validator = ConsistencyValidator()
+    source_cache = Path(args.source_cache) if args.source_cache else None
+    result = validator.validate(module_dir, source_cache=source_cache)
+
+    if result.warnings:
+        print("\nConsistency warnings:")
+        for w in result.warnings:
+            print(f"  ⚠️ {w}")
+
+    status = "ok" if result.passed else "fail"
+    print(json.dumps({"status": status, "errors": len(result.errors),
+                      "warnings": len(result.warnings),
+                      "progress": progress or {}}, ensure_ascii=False))
+
+
+def cmd_links(args: argparse.Namespace) -> None:
+    """Run link/embedding health check."""
+    from core.validators.links import LinksValidator
+
+    module_dir = Path(args.module_dir) if args.module_dir else None
+    if not module_dir:
+        print("Error: --module-dir required", file=sys.stderr)
+        sys.exit(1)
+
+    validator = LinksValidator()
+    result = validator.validate(module_dir)
+
+    if result.errors:
+        print("Broken links:")
+        for e in result.errors:
+            print(f"  ❌ {e}")
+    if result.warnings:
+        for w in result.warnings:
+            print(f"  ⚠️ {w}")
+
+    status = "ok" if result.passed else "fail"
+    print(json.dumps({"status": status, "errors": len(result.errors),
+                      "warnings": len(result.warnings)}, ensure_ascii=False))
+
+
+def cmd_structure(args: argparse.Namespace) -> None:
+    """Run structure/output validation."""
+    from core.validators.structure import StructureValidator
+
+    module_dir = Path(args.module_dir)
+    skeleton_dir = Path(args.skeleton_dir) if args.skeleton_dir else None
+
+    validator = StructureValidator()
+    result = validator.validate(module_dir, skeleton_dir=skeleton_dir)
+
+    if result.errors:
+        print("Structure errors:")
+        for e in result.errors:
+            print(f"  ❌ {e}")
+    if result.warnings:
+        for w in result.warnings:
+            print(f"  ⚠️ {w}")
+
+    status = "ok" if result.passed else "fail"
+    print(json.dumps({"status": status, "errors": len(result.errors),
+                      "warnings": len(result.warnings)}, ensure_ascii=False))
+
+
+def cmd_sampling(args: argparse.Namespace) -> None:
+    """Run numeric sampling validation (spot-check enum values and field names)."""
+    from core.validators.sampling import SamplingValidator
+
+    module_dir = Path(args.module_dir)
+    validator = SamplingValidator()
+    result = validator.validate(module_dir)
+
+    if result.warnings:
+        print("Sampling warnings:")
+        for w in result.warnings:
+            print(f"  ⚠️ {w}")
+
+    hit_count = 10 - len(result.warnings)  # max 10 samples (5 enum + 5 field)
+    total = 10
+    hit_rate = round(hit_count / total * 100, 1) if total > 0 else 100.0
+
+    if not result.warnings:
+        print(f"Numeric sampling validation passed (hit rate {hit_rate}%)")
+    else:
+        print(f"Numeric sampling hit rate {hit_rate}% ({len(result.warnings)} items missed)")
+
+    status = "ok" if result.passed else "warn"
+    print(json.dumps({"status": status, "warnings": len(result.warnings),
+                      "hit_rate_pct": hit_rate}, ensure_ascii=False), file=sys.stderr)
+
+
+def main():
+    parser = argparse.ArgumentParser(prog="python -m core.validators", description="Validation tools")
+    sub = parser.add_subparsers(dest="command")
+
+    # coverage
+    p = sub.add_parser("coverage", help="Coverage check")
+    p.add_argument("action", nargs="?", default="check", choices=["check"])
+    p.add_argument("--skeleton", help="Skeleton JSON path")
+    p.add_argument("--skeleton-dir", help="Skeleton shards directory")
+    p.add_argument("--module-dir", help="Module directory")
+    p.add_argument("--docs-dir", help="Documents directory")
+    p.add_argument("--type", default="service", help="Module type")
+
+    # consistency (replaces check_progress)
+    p = sub.add_parser("consistency", help="Consistency and progress check")
+    p.add_argument("--module-dir", required=True, help="Module directory")
+    p.add_argument("--preset", help="Preset name")
+    p.add_argument("--source-cache", help="Source cache path")
+    p.add_argument("--cleanup", action="store_true", help="Clean up progress files")
+
+    # links (replaces embedding_health)
+    p = sub.add_parser("links", help="Link and reference health check")
+    p.add_argument("--module-dir", help="Module directory")
+    p.add_argument("--config", help="kb-project.yaml path")
+
+    # structure (replaces validate_output)
+    p = sub.add_parser("structure", help="Document structure validation")
+    p.add_argument("--module-dir", required=True, help="Module directory")
+    p.add_argument("--skeleton-dir", help="Skeleton directory for comparison")
+    p.add_argument("--kb-dir", help="Knowledge base directory (validate all modules)")
+
+    # sampling (numeric spot-check)
+    p = sub.add_parser("sampling", help="Numeric sampling validation (enum values + field names)")
+    p.add_argument("--module-dir", required=True, help="Module directory")
+
+    args = parser.parse_args()
+    if not args.command:
+        parser.print_help()
+        sys.exit(1)
+
+    commands = {
+        "coverage": cmd_coverage, "consistency": cmd_consistency,
+        "links": cmd_links, "structure": cmd_structure,
+        "sampling": cmd_sampling,
+    }
+    commands[args.command](args)
+
+
+if __name__ == "__main__":
+    main()