source-kb 0.2.2__py3-none-any.whl

core/skeleton/extract_methods.py

@@ -0,0 +1,298 @@
+"""Method-level source extraction — complexity-graded content extraction.
+
+Extracts source at method granularity: high=full body, medium=signature+20 lines,
+low=signature only. Falls back to head truncation when tree-sitter is unavailable.
+
+Usage:
+    from core.skeleton.extract_methods import extract_methods_from_file, is_available
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def is_available() -> bool:
+    """Check if tree-sitter is available for AST-based extraction."""
+    try:
+        import tree_sitter_languages  # noqa: F401
+        return True
+    except ImportError:
+        return False
+
+
+def extract_methods_from_file(
+    content: str,
+    filepath: str | Path,
+    max_bytes: int = 30_000,
+    priority_info: dict[str, Any] | None = None,
+) -> str:
+    """Extract content from a source file at method granularity.
+
+    Returns formatted method-level content or head-truncated fallback.
+    """
+    if not content:
+        return ""
+
+    content_bytes = len(content.encode("utf-8"))
+
+    # File within budget — return as-is
+    if content_bytes <= max_bytes:
+        return content
+
+    # Try tree-sitter method-level extraction
+    if is_available():
+        extracted = _extract_by_methods(content, filepath, priority_info, max_bytes)
+        if extracted:
+            return extracted
+
+    # Fallback: head truncation
+    return _truncate_head(content, max_bytes)
+
+
+# ---------------------------------------------------------------------------
+# Internal: tree-sitter extraction
+# ---------------------------------------------------------------------------
+
+
+def _extract_by_methods(
+    content: str, filepath: str | Path,
+    priority_info: dict[str, Any] | None, max_bytes: int,
+) -> str | None:
+    """Use tree-sitter for method-level extraction."""
+    try:
+        import tree_sitter_languages
+    except ImportError:
+        return None
+
+    lang = _detect_language(str(filepath))
+    if not lang:
+        return None
+
+    try:
+        parser = tree_sitter_languages.get_parser(lang)
+        tree = parser.parse(content.encode("utf-8"))
+    except Exception as e:
+        logger.debug("tree-sitter parse failed for %s: %s", filepath, e)
+        return None
+
+    lines = content.splitlines(keepends=True)
+    methods = _find_methods(tree.root_node, lang)
+    if not methods:
+        return None
+
+    # Get high-priority method names from skeleton
+    high_method_names: set[str] = set()
+    if priority_info:
+        for m in priority_info.get("method_details", []):
+            if m.get("complexity") == "high":
+                high_method_names.add(m.get("name", ""))
+
+    # Classify methods
+    high_methods = []
+    medium_methods = []
+    low_methods = []
+
+    for m in methods:
+        if m["name"] in high_method_names or m["complexity"] == "high":
+            high_methods.append(m)
+        elif m["complexity"] == "medium":
+            medium_methods.append(m)
+        else:
+            low_methods.append(m)
+
+    # Build output
+    parts: list[str] = []
+    total = 0
+
+    # 1. File header (package/import + class declaration)
+    header_end = _find_header_end(lines, lang)
+    header = "".join(lines[:header_end])
+    header_bytes = len(header.encode("utf-8"))
+    if header_bytes < max_bytes * 0.3:
+        parts.append(header)
+        total += header_bytes
+    else:
+        header = "".join(lines[:20])
+        parts.append(header + "\n// ... [imports truncated]\n\n")
+        total += len(header.encode("utf-8")) + 30
+
+    # 2. High methods: full body
+    for m in high_methods:
+        body = "".join(lines[m["start"]:m["end"] + 1])
+        body_bytes = len(body.encode("utf-8"))
+        if total + body_bytes > max_bytes:
+            remaining = max_bytes - total - 100
+            if remaining > 500:
+                truncated = body.encode("utf-8")[:remaining].decode("utf-8", errors="ignore")
+                parts.append(f"\n    // === HIGH: {m['name']} (truncated) ===\n{truncated}\n    // ...\n")
+                total = max_bytes
+            break
+        parts.append(f"\n    // === HIGH: {m['name']} ({m['line_count']} lines) ===\n{body}")
+        total += body_bytes
+
+    # 3. Medium methods: signature + first 20 lines
+    for m in medium_methods:
+        if total >= max_bytes:
+            break
+        preview_end = min(m["start"] + 20, m["end"] + 1)
+        body = "".join(lines[m["start"]:preview_end])
+        suffix = ""
+        if preview_end < m["end"] + 1:
+            suffix = f"\n        // ... [{m['line_count'] - 20} more lines]\n"
+        block = f"\n    // === MEDIUM: {m['name']} ({m['line_count']} lines) ===\n" + body + suffix
+        block_bytes = len(block.encode("utf-8"))
+        if total + block_bytes > max_bytes:
+            break
+        parts.append(block)
+        total += block_bytes
+
+    # 4. Low methods: signatures only
+    if low_methods and total < max_bytes - 500:
+        sig_lines = ["\n    // === LOW complexity methods (signatures only) ===\n"]
+        for m in low_methods:
+            sig = "".join(lines[m["start"]:m["start"] + 1]).rstrip()
+            sig_lines.append(f"    {sig}\n")
+        sig_block = "".join(sig_lines)
+        sig_bytes = len(sig_block.encode("utf-8"))
+        if total + sig_bytes <= max_bytes:
+            parts.append(sig_block)
+
+    result = "".join(parts)
+    if len(result.encode("utf-8")) < 200:
+        return None
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Internal: Method finding
+# ---------------------------------------------------------------------------
+
+_METHOD_NODE_TYPES: dict[str, list[str]] = {
+    "java": ["method_declaration", "constructor_declaration"],
+    "python": ["function_definition"], "go": ["function_declaration", "method_declaration"],
+    "typescript": ["function_declaration", "method_definition"],
+    "javascript": ["function_declaration", "method_definition"],
+    "kotlin": ["function_declaration"], "rust": ["function_item"],
+}
+
+
+def _find_methods(root_node: Any, lang: str) -> list[dict[str, Any]]:
+    """Extract method info from AST."""
+    node_types = _METHOD_NODE_TYPES.get(lang, [])
+    methods: list[dict] = []
+
+    def walk(node: Any) -> None:
+        if node.type in node_types:
+            name = _get_method_name(node)
+            if name:
+                start = node.start_point[0]
+                end = node.end_point[0]
+                line_count = end - start + 1
+                branches = _count_branches(node)
+                if line_count > 50 or branches > 10:
+                    complexity = "high"
+                elif line_count > 25 or branches > 5:
+                    complexity = "medium"
+                else:
+                    complexity = "low"
+                methods.append({
+                    "name": name, "start": start, "end": end,
+                    "line_count": line_count, "complexity": complexity,
+                })
+        for child in node.children:
+            walk(child)
+
+    walk(root_node)
+    return methods
+
+
+def _get_method_name(node: Any) -> str | None:
+    """Extract method name from AST node."""
+    for child in node.children:
+        if child.type in ("identifier", "name"):
+            return child.text.decode("utf-8") if child.text else None
+        if child.type == "constructor_declarator":
+            for sub in child.children:
+                if sub.type == "identifier":
+                    return sub.text.decode("utf-8") if sub.text else None
+    return None
+
+
+def _count_branches(node: Any) -> int:
+    """Count branch statements in a method."""
+    branch_types = {"if_statement", "else_clause", "switch_expression", "for_statement",
+                    "while_statement", "try_statement", "catch_clause",
+                    "ternary_expression", "conditional_expression", "enhanced_for_statement"}
+    count = 0
+
+    def walk(n: Any) -> None:
+        nonlocal count
+        if n.type in branch_types:
+            count += 1
+        for child in n.children:
+            walk(child)
+
+    walk(node)
+    return count
+
+
+# ---------------------------------------------------------------------------
+# Internal: Helpers
+# ---------------------------------------------------------------------------
+
+_LANG_EXTENSIONS: dict[str, str] = {
+    ".java": "java", ".py": "python", ".go": "go", ".ts": "typescript",
+    ".tsx": "typescript", ".js": "javascript", ".jsx": "javascript",
+    ".kt": "kotlin", ".rs": "rust",
+}
+
+
+def _detect_language(filepath: str) -> str | None:
+    """Detect language from file extension."""
+    ext = Path(filepath).suffix.lower()
+    return _LANG_EXTENSIONS.get(ext)
+
+
+def _find_header_end(lines: list[str], lang: str) -> int:
+    """Find end of file header (package/import section)."""
+    if lang == "java":
+        last_import = 0
+        for i, line in enumerate(lines):
+            stripped = line.strip()
+            if stripped.startswith(("import ", "package ")):
+                last_import = i
+            if any(kw in stripped for kw in ("class ", "interface ", "enum ")):
+                if i > last_import + 1:
+                    return i
+        return min(last_import + 2, len(lines))
+    for i, line in enumerate(lines[:50]):
+        stripped = line.strip()
+        if lang == "python" and stripped.startswith(("def ", "class ")):
+            return i
+        if lang in ("go", "rust") and stripped.startswith("func "):
+            return i
+    return min(30, len(lines))
+
+
+def _truncate_head(content: str, max_bytes: int) -> str:
+    """Fallback: truncate to budget, preserving file head."""
+    lines = content.splitlines(keepends=True)
+    result: list[str] = []
+    acc = 0
+    for line in lines:
+        acc += len(line.encode("utf-8"))
+        if acc > max_bytes:
+            break
+        result.append(line)
+    return "".join(result) + f"\n// ... [truncated, showing {len(result)}/{len(lines)} lines]\n"

core/skeleton/file_list.py

@@ -0,0 +1,239 @@
+"""File list extraction and coverage checking.
+
+Generates per-doc-type file lists from skeleton entries using preset classification rules.
+Also provides coverage checking to identify unclassified files.
+
+Usage:
+    from core.skeleton.file_list import extract_file_list, check_coverage, load_skeleton
+
+    entries = load_skeleton(module_dir)
+    files = extract_file_list(entries, preset, "business-logic", source_cache)
+    report = check_coverage(entries, preset, file_lists_dir)
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Skeleton loading
+# ---------------------------------------------------------------------------
+
+
+def load_skeleton(module_dir: Path) -> list[dict[str, Any]]:
+    """Load skeleton entries from module directory (auto-detects format).
+
+    Handles: single file, shards directory, legacy paths.
+    Falls back to shards if single file is empty/corrupted.
+    """
+    from core.paths import resolve_skeleton
+
+    resolved = resolve_skeleton(module_dir)
+    if resolved is None:
+        return []
+
+    if resolved.is_dir():
+        return _load_dir(resolved)
+
+    # Single file
+    try:
+        data = json.loads(resolved.read_text(encoding="utf-8"))
+        entries = data if isinstance(data, list) else data.get("data", data.get("files", []))
+        if entries:
+            return entries
+    except (json.JSONDecodeError, UnicodeDecodeError, OSError):
+        pass
+
+    # Fallback to shards directory
+    shards_dir = resolved.parent / "shards"
+    if shards_dir.is_dir():
+        return _load_dir(shards_dir)
+    return []
+
+
+def _load_dir(skeleton_dir: Path) -> list[dict]:
+    """Load all JSON shards from a directory."""
+    entries: list[dict] = []
+    for f in sorted(skeleton_dir.glob("*.json")):
+        try:
+            data = json.loads(f.read_text(encoding="utf-8"))
+            if isinstance(data, list):
+                entries.extend(data)
+            elif isinstance(data, dict):
+                entries.extend(data.get("data", data.get("files", [])))
+        except (json.JSONDecodeError, OSError):
+            continue
+    return entries
+
+
+# ---------------------------------------------------------------------------
+# File list extraction
+# ---------------------------------------------------------------------------
+
+
+def extract_file_list(
+    entries: list[dict],
+    preset: dict,
+    doc_type: str,
+    source_cache: str | Path,
+) -> list[str]:
+    """Extract absolute file paths for a given doc type.
+
+    Uses preset classification rules to determine which files belong to each doc type.
+    Returns sorted list of absolute paths (forward-slash normalized).
+    """
+    from core.preset import classify_file, get_affected_docs, get_doc_filename
+
+    doc_filename = get_doc_filename(preset, doc_type, strict=False)
+    enums_filename = get_doc_filename(preset, "enums-and-constants", strict=False)
+    matched: set[str] = set()
+
+    for entry in entries:
+        file_path = entry.get("file", "")
+        if not file_path:
+            continue
+
+        categories = classify_file(preset, file_path, entry)
+        affected_docs = get_affected_docs(preset, categories)
+
+        if doc_filename not in affected_docs:
+            continue
+
+        # Enum purity check: mixed files (class + inner enum) go to data-models
+        if doc_filename == enums_filename:
+            has_enum = "enum" in categories
+            has_constant = "constant" in categories
+            if has_enum and not has_constant and not is_pure_enum_file(entry):
+                continue
+
+        matched.add(file_path)
+
+    return sorted(
+        p.replace("\\", "/")
+        for p in matched
+    )
+
+
+def is_pure_enum_file(entry: dict) -> bool:
+    """Check if all classes in the entry are enum type."""
+    classes = entry.get("classes", [])
+    if not classes:
+        return False
+    return all(c.get("type", "").lower() == "enum" for c in classes)
+
+
+# ---------------------------------------------------------------------------
+# Coverage checking
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class CoverageReport:
+    """Result of a coverage check."""
+    total_files: int = 0
+    covered: int = 0
+    uncovered_files: list[str] = field(default_factory=list)
+    distribution: dict[str, int] = field(default_factory=dict)
+
+    @property
+    def uncovered_count(self) -> int:
+        return len(self.uncovered_files)
+
+    @property
+    def coverage_pct(self) -> float:
+        return (self.covered / self.total_files * 100) if self.total_files > 0 else 100.0
+
+
+def check_coverage(
+    entries: list[dict],
+    preset: dict,
+    file_lists_dir: Path | None = None,
+) -> CoverageReport:
+    """Check which files are not covered by any doc-type classification.
+
+    A file is considered covered if:
+    1. Preset classification rules match it to at least one meaningful category, OR
+    2. It already appears in a file-list (manually added)
+
+    Args:
+        entries: Skeleton entries
+        preset: Preset configuration
+        file_lists_dir: Directory containing existing .txt file lists (for manual coverage)
+
+    Returns:
+        CoverageReport with uncovered files and distribution stats
+    """
+    from core.preset import classify_file, get_affected_docs, get_file_classification
+
+    classification = get_file_classification(preset)
+    skip_categories = {
+        name for name, cfg in classification.items()
+        if cfg.get("trigger") == "add-delete-only"
+    }
+
+    # Load existing file-list paths for manual coverage detection
+    manual_covered = _load_file_list_paths(file_lists_dir) if file_lists_dir else set()
+
+    report = CoverageReport(total_files=len(entries))
+    distribution: dict[str, int] = {}
+
+    for entry in entries:
+        file_path = entry.get("file", "")
+        if not file_path:
+            continue
+
+        categories = classify_file(preset, file_path, entry)
+        meaningful = [c for c in categories if c not in skip_categories]
+
+        if not meaningful:
+            if _is_in_file_lists(file_path, manual_covered):
+                distribution["(manual)"] = distribution.get("(manual)", 0) + 1
+            else:
+                report.uncovered_files.append(file_path)
+                continue
+
+        affected = get_affected_docs(preset, meaningful)
+        for doc in affected:
+            distribution[doc] = distribution.get(doc, 0) + 1
+
+    report.covered = report.total_files - report.uncovered_count
+    report.distribution = distribution
+    return report
+
+
+def _load_file_list_paths(file_lists_dir: Path) -> set[str]:
+    """Load all file paths from existing file-list .txt files."""
+    covered: set[str] = set()
+    if not file_lists_dir.is_dir():
+        return covered
+    for f in file_lists_dir.iterdir():
+        if not f.name.endswith(".txt") or f.name.startswith("coverage"):
+            continue
+        for line in f.read_text(encoding="utf-8").splitlines():
+            line = line.strip()
+            if line:
+                covered.add(line)
+                # Also store module-relative path
+                parts = line.replace("\\", "/").split(".source-cache/")
+                if len(parts) > 1:
+                    module_rel = "/".join(parts[-1].split("/")[1:])
+                    covered.add(module_rel)
+    return covered
+
+
+def _is_in_file_lists(file_path: str, covered: set[str]) -> bool:
+    """Check if a skeleton file path is already in file-lists."""
+    normalized = file_path.replace("\\", "/")
+    if normalized in covered:
+        return True
+    for p in covered:
+        if p.endswith(normalized) or normalized.endswith(p):
+            return True
+    return False