source-kb 0.2.2__py3-none-any.whl

core/skeleton/parsers/regex.py

@@ -0,0 +1,171 @@
+"""Regex-based skeleton parser — zero-dependency fallback.
+
+Uses language-specific regex patterns (loaded from preset config) to extract
+class/method declarations. Less accurate than tree-sitter but works everywhere.
+
+Pattern configuration comes from preset.yaml → parser_config.regex_rules.
+If no rules configured, uses built-in defaults for common languages.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from pathlib import Path
+from typing import Any
+
+from core.interfaces import SkeletonParser
+from core.skeleton.parsers import register_parser
+
+logger = logging.getLogger(__name__)
+
+# Built-in regex patterns per language (used when preset doesn't define custom rules)
+DEFAULT_PATTERNS: dict[str, dict[str, str]] = {
+    "java": {
+        "class": r"^\s*(?:(?:public|protected|private|abstract|final|static)\s+)*(?:class|interface|enum|record)\s+(\w+)",
+        "method": r"^\s*(?:(?:public|protected|private|static|final|abstract|synchronized|native)\s+)*[\w<>\[\],\s]+\s+(\w+)\s*\(",
+        "field": r"^\s*(?:(?:public|protected|private|static|final|volatile|transient)\s+)*[\w<>\[\],\s]+\s+(\w+)\s*[;=]",
+    },
+    "python": {
+        "class": r"^\s*class\s+(\w+)",
+        "method": r"^\s*(?:async\s+)?def\s+(\w+)\s*\(",
+    },
+    "go": {
+        "class": r"^type\s+(\w+)\s+struct\b",
+        "method": r"^func\s+(?:\(\w+\s+\*?\w+\)\s+)?(\w+)\s*\(",
+    },
+    "typescript": {
+        "class": r"^\s*(?:export\s+)?(?:abstract\s+)?(?:class|interface|enum)\s+(\w+)",
+        "method": r"^\s*(?:(?:public|private|protected|static|async|abstract)\s+)*(\w+)\s*\(",
+    },
+    "kotlin": {
+        "class": r"^\s*(?:(?:open|abstract|data|sealed|internal|private)\s+)*(?:class|interface|object|enum\s+class)\s+(\w+)",
+        "method": r"^\s*(?:(?:override|open|private|internal|suspend)\s+)*fun\s+(\w+)\s*\(",
+    },
+}
+
+# Extension → language key
+EXT_TO_LANG: dict[str, str] = {
+    ".java": "java", ".py": "python", ".go": "go",
+    ".ts": "typescript", ".tsx": "typescript", ".js": "typescript",
+    ".kt": "kotlin", ".kts": "kotlin",
+    ".cs": "java",  # C# patterns similar enough to Java for basic extraction
+    ".scala": "java",
+}
+
+BRANCH_PATTERN = re.compile(
+    r"^\s*(?:if|else|elif|switch|case|match|for|while|catch|except|when)\b"
+)
+
+
+@register_parser
+class RegexSkeletonParser(SkeletonParser):
+    """Regex-based fallback parser. Registered as 'regex'. Lowest priority."""
+
+    name = "regex"
+    priority = 50
+
+    def can_parse(self, repo_path: Path, preset: dict[str, Any]) -> bool:
+        """Always available — zero dependencies."""
+        return True
+
+    def parse(self, repo_path: Path, preset: dict[str, Any], **kwargs: Any) -> list[dict[str, Any]]:
+        """Parse source files using regex patterns from preset config."""
+        from core.git import ls_tree
+
+        ref = kwargs.get("ref", "HEAD")
+        subpath = kwargs.get("subpath")
+        parser_config = preset.get("parser_config", {})
+        extensions = parser_config.get("extensions") or list(EXT_TO_LANG.keys())
+        skip_dirs = set(parser_config.get("skip_dirs", ["test", "tests", "generated"]))
+        custom_rules = parser_config.get("regex_rules", {})
+
+        all_files = ls_tree(repo_path, ref)
+
+        subpath_prefix = ""
+        if subpath:
+            subpath_prefix = subpath.rstrip("/") + "/"
+            all_files = [f for f in all_files if f.startswith(subpath_prefix)]
+
+        entries: list[dict[str, Any]] = []
+
+        for filepath in all_files:
+            rel_path = filepath[len(subpath_prefix):] if subpath_prefix else filepath
+            if any(f"/{d}/" in rel_path or rel_path.startswith(f"{d}/") for d in skip_dirs):
+                continue
+            if not any(rel_path.endswith(ext) for ext in extensions):
+                continue
+
+            content = _read_file(repo_path, ref, filepath)
+            if content is None:
+                continue
+
+            entry = _parse_file(content, rel_path, custom_rules)
+            if entry:
+                entries.append(entry)
+
+        return entries
+
+
+def _read_file(repo_path: Path, ref: str, filepath: str) -> str | None:
+    """Read file content from git or filesystem."""
+    from core.git import read_file
+    try:
+        return read_file(repo_path, ref, filepath)
+    except Exception:
+        full = repo_path / filepath
+        if full.exists():
+            try:
+                return full.read_text(encoding="utf-8")
+            except (OSError, UnicodeDecodeError):
+                pass
+    return None
+
+
+def _parse_file(content: str, filepath: str, custom_rules: dict) -> dict[str, Any] | None:
+    """Parse a single file using regex patterns."""
+    ext = Path(filepath).suffix.lower()
+    lang = EXT_TO_LANG.get(ext)
+    if not lang:
+        return None
+
+    # Use custom rules if provided, otherwise built-in defaults
+    patterns = custom_rules if custom_rules else DEFAULT_PATTERNS.get(lang, {})
+    if not patterns:
+        return None
+
+    lines = content.splitlines()
+    total_lines = len(lines)
+
+    class_re = re.compile(patterns["class"]) if "class" in patterns else None
+    method_re = re.compile(patterns["method"]) if "method" in patterns else None
+
+    classes: list[dict] = []
+    method_starts: list[tuple[str, int]] = []
+
+    for i, line in enumerate(lines):
+        if class_re:
+            m = class_re.match(line)
+            if m:
+                classes.append({"name": m.group(1)})
+        if method_re:
+            m = method_re.match(line)
+            if m:
+                method_starts.append((m.group(1), i))
+
+    # Compute method metrics
+    methods: list[dict] = []
+    for idx, (name, start) in enumerate(method_starts):
+        end = method_starts[idx + 1][1] if idx + 1 < len(method_starts) else total_lines
+        line_count = end - start
+        branches = sum(1 for l in lines[start:end] if BRANCH_PATTERN.match(l))
+        complexity = "high" if line_count > 50 or branches > 10 else "medium" if line_count > 25 or branches > 5 else "low"
+        methods.append({
+            "name": name, "line_count": line_count,
+            "complexity": complexity, "branches": branches,
+        })
+
+    if not methods and not classes:
+        return None
+
+    return {"file": filepath, "total_lines": total_lines, "methods": methods, "classes": classes}

core/skeleton/parsers/treesitter.py

@@ -0,0 +1,324 @@
+"""Tree-sitter based skeleton parser — multi-language AST extraction.
+
+Wraps tree-sitter-languages to extract classes, methods, fields, imports,
+and call targets from source files. Language-specific rules come from preset config.
+"""
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any
+
+from core.interfaces import SkeletonParser
+from core.skeleton.parsers import register_parser
+
+logger = logging.getLogger(__name__)
+
+# Extension → tree-sitter language name
+EXT_TO_LANG: dict[str, str] = {
+    ".java": "java", ".py": "python", ".go": "go",
+    ".ts": "typescript", ".tsx": "tsx", ".js": "javascript", ".jsx": "javascript",
+    ".kt": "kotlin", ".kts": "kotlin", ".scala": "scala", ".rs": "rust",
+    ".rb": "ruby", ".cs": "c_sharp", ".cpp": "cpp", ".cc": "cpp", ".cxx": "cpp",
+    ".c": "c", ".h": "c", ".hpp": "cpp", ".swift": "swift", ".dart": "dart",
+    ".lua": "lua", ".php": "php", ".groovy": "groovy", ".r": "r", ".R": "r",
+}
+
+# tree-sitter node types for classes/structs per language
+CLASS_NODES: dict[str, list[str]] = {
+    "java": ["class_declaration", "interface_declaration", "enum_declaration", "record_declaration"],
+    "python": ["class_definition"], "go": ["type_declaration"],
+    "typescript": ["class_declaration", "interface_declaration", "enum_declaration"],
+    "tsx": ["class_declaration", "interface_declaration", "enum_declaration"],
+    "javascript": ["class_declaration"],
+    "kotlin": ["class_declaration", "object_declaration", "interface_declaration"],
+    "scala": ["class_definition", "object_definition", "trait_definition"],
+    "rust": ["struct_item", "enum_item", "impl_item", "trait_item"],
+    "ruby": ["class", "module"],
+    "c_sharp": ["class_declaration", "interface_declaration", "enum_declaration",
+                "struct_declaration", "record_declaration"],
+    "cpp": ["class_specifier", "struct_specifier"], "c": ["struct_specifier"],
+    "swift": ["class_declaration", "struct_declaration", "protocol_declaration", "enum_declaration"],
+    "php": ["class_declaration", "interface_declaration", "enum_declaration"],
+}
+
+# tree-sitter node types for functions/methods per language
+METHOD_NODES: dict[str, list[str]] = {
+    "java": ["method_declaration", "constructor_declaration"],
+    "python": ["function_definition"],
+    "go": ["function_declaration", "method_declaration"],
+    "typescript": ["function_declaration", "method_definition", "arrow_function"],
+    "tsx": ["function_declaration", "method_definition", "arrow_function"],
+    "javascript": ["function_declaration", "method_definition", "arrow_function"],
+    "kotlin": ["function_declaration"], "scala": ["function_definition", "val_definition"],
+    "rust": ["function_item"], "ruby": ["method", "singleton_method"],
+    "c_sharp": ["method_declaration", "constructor_declaration"],
+    "cpp": ["function_definition"], "c": ["function_definition"],
+    "swift": ["function_declaration"], "php": ["method_declaration", "function_definition"],
+}
+
+# Branch node types for complexity analysis
+BRANCH_TYPES = frozenset({
+    "if_statement", "if_expression", "else_clause", "elif_clause",
+    "switch_statement", "switch_expression", "case_clause", "match_expression",
+    "match_arm", "for_statement", "for_expression", "while_statement",
+    "try_statement", "catch_clause", "except_clause",
+    "ternary_expression", "conditional_expression", "guard_statement",
+})
+
+
+
+def is_available() -> bool:
+    """Check if tree-sitter-languages is installed."""
+    try:
+        import tree_sitter_languages  # noqa: F401
+        return True
+    except ImportError:
+        return False
+
+
+def detect_language(filepath: str) -> str | None:
+    """Detect language from file extension."""
+    return EXT_TO_LANG.get(Path(filepath).suffix.lower())
+
+@register_parser
+class TreeSitterSkeletonParser(SkeletonParser):
+    """Tree-sitter skeleton parser. Registered as 'treesitter'."""
+
+    name = "treesitter"
+    priority = 80
+
+    def can_parse(self, repo_path: Path, preset: dict[str, Any]) -> bool:
+        return is_available()
+
+    def parse(self, repo_path: Path, preset: dict[str, Any], **kwargs: Any) -> list[dict[str, Any]]:
+        """Parse all source files in repo, returning skeleton entries."""
+        from core.git import ls_tree
+        ref = kwargs.get("ref", "HEAD")
+        subpath = kwargs.get("subpath")
+        parser_config = preset.get("parser_config", {})
+        extensions = parser_config.get("extensions") or list(EXT_TO_LANG.keys())
+        skip_dirs = set(parser_config.get("skip_dirs", ["test", "tests", "generated"]))
+        all_files = ls_tree(repo_path, ref)
+
+        subpath_prefix = ""
+        if subpath:
+            subpath_prefix = subpath.rstrip("/") + "/"
+            all_files = [f for f in all_files if f.startswith(subpath_prefix)]
+
+        entries: list[dict[str, Any]] = []
+        for filepath in all_files:
+            rel_path = filepath[len(subpath_prefix):] if subpath_prefix else filepath
+            if any(f"/{d}/" in rel_path or rel_path.startswith(f"{d}/") for d in skip_dirs):
+                continue
+            if not any(rel_path.endswith(ext) for ext in extensions):
+                continue
+            content = _read_file_content(repo_path, ref, filepath)
+            if content is None:
+                continue
+            entry = parse_file(content, rel_path)
+            if entry:
+                entries.append(entry)
+        return entries
+
+    def list_source_files(self, repo_path: Path, preset: dict[str, Any], ref: str = "HEAD", **kwargs: Any) -> list[str]:
+        """List source files (excluding test and generated files)."""
+        from core.git import ls_tree
+        subpath = kwargs.get("subpath")
+        parser_config = preset.get("parser_config", {})
+        extensions = parser_config.get("extensions") or list(EXT_TO_LANG.keys())
+        test_patterns = parser_config.get("test_patterns", [])
+        generated_patterns = parser_config.get("generated_patterns", [])
+        all_files = ls_tree(repo_path, ref)
+
+        subpath_prefix = ""
+        if subpath:
+            subpath_prefix = subpath.rstrip("/") + "/"
+            all_files = [f for f in all_files if f.startswith(subpath_prefix)]
+
+        result = []
+        for f in all_files:
+            rel_path = f[len(subpath_prefix):] if subpath_prefix else f
+            if any(p in rel_path for p in test_patterns):
+                continue
+            if any(p in rel_path for p in generated_patterns):
+                continue
+            if extensions and not any(rel_path.endswith(ext) for ext in extensions):
+                continue
+            result.append(rel_path)
+        return result
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+_parser_cache: dict[str, Any] = {}
+_parser_lock = __import__("threading").Lock()
+
+def _get_ts_parser(lang: str):
+    """Get or create a tree-sitter parser for a language."""
+    with _parser_lock:
+        if lang in _parser_cache:
+            return _parser_cache[lang]
+    try:
+        import tree_sitter_languages
+        parser = tree_sitter_languages.get_parser(lang)
+        with _parser_lock:
+            _parser_cache[lang] = parser
+        return parser
+    except (ImportError, Exception) as e:
+        logger.warning("No tree-sitter grammar for %s: %s", lang, e)
+        return None
+
+
+def _read_file_content(repo_path: Path, ref: str, filepath: str) -> str | None:
+    """Read file content from git or filesystem."""
+    from core.git import read_file
+    try:
+        return read_file(repo_path, ref, filepath)
+    except Exception:
+        full = repo_path / filepath
+        if full.exists():
+            try:
+                return full.read_text(encoding="utf-8")
+            except (OSError, UnicodeDecodeError):
+                pass
+    return None
+
+
+def parse_file(content: str, filepath: str) -> dict[str, Any] | None:
+    """Parse a single file into a skeleton entry. Public for direct use."""
+    from core.skeleton.parsers.treesitter_java import extract_call_targets
+    from core.skeleton.parsers.treesitter_multi import (
+        fallback_parse, extract_imports, extract_classes, extract_methods, extract_fields,
+    )
+    lang = detect_language(filepath)
+    if not lang:
+        return fallback_parse(content, filepath)
+    parser = _get_ts_parser(lang)
+    if not parser:
+        return fallback_parse(content, filepath)
+    try:
+        tree = parser.parse(content.encode("utf-8"))
+    except Exception as e:
+        logger.debug("Parse failed for %s: %s", filepath, e)
+        return fallback_parse(content, filepath)
+
+    root = tree.root_node
+    classes = extract_classes(root, lang)
+    methods = extract_methods(root, lang, content)
+    if not methods and not classes:
+        return None
+    entry: dict[str, Any] = {
+        "file": filepath, "total_lines": len(content.splitlines()),
+        "methods": methods, "classes": classes,
+    }
+    fields = extract_fields(root, lang)
+    if fields:
+        entry["fields"] = fields
+    imports = extract_imports(root, lang)
+    if imports:
+        entry["imports"] = imports
+    calls = extract_call_targets(root, lang, content)
+    if calls:
+        entry["calls"] = calls
+    return entry
+
+
+# ---------------------------------------------------------------------------
+# Common AST traversal utilities (used by treesitter_java and treesitter_multi)
+# ---------------------------------------------------------------------------
+
+def _walk_nodes(root, node_types: list[str]):
+    """Recursively yield nodes matching given types."""
+    stack = [root]
+    while stack:
+        node = stack.pop()
+        if node.type in node_types:
+            yield node
+        stack.extend(reversed(node.children))
+
+def _get_name(node, lang: str) -> str | None:
+    """Extract name identifier from a declaration node."""
+    name_node = node.child_by_field_name("name")
+    if name_node:
+        return name_node.text.decode("utf-8")
+    for child in node.children:
+        if child.type in ("identifier", "type_identifier", "property_identifier"):
+            return child.text.decode("utf-8")
+    return None
+
+def _classify_type(node, lang: str) -> str:
+    """Classify a class-like node into class/interface/enum/struct."""
+    _type_map = {
+        "interface_declaration": "interface", "enum_declaration": "enum",
+        "record_declaration": "record", "struct_item": "struct",
+        "struct_specifier": "struct", "struct_declaration": "struct",
+        "trait_item": "interface", "trait_definition": "interface",
+        "object_declaration": "object", "object_definition": "object",
+        "protocol_declaration": "interface", "module": "module",
+    }
+    return _type_map.get(node.type, "class")
+
+def _get_access(node, lang: str) -> str:
+    """Extract access modifier."""
+    if lang in ("java", "c_sharp", "kotlin"):
+        for child in node.children:
+            if child.type == "modifiers":
+                text = child.text.decode("utf-8")
+                for mod in ("public", "protected", "private"):
+                    if mod in text:
+                        return mod
+    elif lang == "go":
+        name = _get_name(node, lang)
+        return "public" if name and name[0].isupper() else "private"
+    elif lang == "rust":
+        if any(c.type == "visibility_modifier" for c in node.children):
+            return "public"
+        return "private"
+    elif lang == "python":
+        name = _get_name(node, lang)
+        if name and name.startswith("__") and not name.endswith("__"):
+            return "private"
+        if name and name.startswith("_"):
+            return "protected"
+    return "public"
+
+def _get_params(node, lang: str) -> str | None:
+    """Extract parameter string."""
+    params_node = (
+        node.child_by_field_name("parameters")
+        or node.child_by_field_name("formal_parameters")
+    )
+    if params_node:
+        text = params_node.text.decode("utf-8").strip("()")
+        return text[:200]
+    return None
+
+def _get_return_type(node, lang: str) -> str | None:
+    """Extract return type."""
+    ret_node = node.child_by_field_name("return_type") or node.child_by_field_name("type")
+    if ret_node:
+        return ret_node.text.decode("utf-8")[:100]
+    return None
+
+def _get_annotations(node, lang: str) -> list[str | dict]:
+    """Extract annotations/decorators."""
+    from core.skeleton.parsers.treesitter_java import parse_java_annotation
+
+    annotations: list[str | dict] = []
+    if lang in ("java", "kotlin", "c_sharp"):
+        for child in node.children:
+            if child.type in ("modifiers", "annotation"):
+                for sub in child.children:
+                    if sub.type in ("marker_annotation", "annotation"):
+                        annotations.append(parse_java_annotation(sub, lang))
+        prev = node.prev_sibling
+        while prev and prev.type in ("marker_annotation", "annotation"):
+            annotations.append(parse_java_annotation(prev, lang))
+            prev = prev.prev_sibling
+    elif lang == "python":
+        for child in node.children:
+            if child.type == "decorator":
+                annotations.append(child.text.decode("utf-8"))
+    return annotations[:10]