source-kb 0.2.2__py3-none-any.whl

core/skeleton/parsers/treesitter_java.py

@@ -0,0 +1,284 @@
+"""Java-specific tree-sitter extraction logic.
+
+Handles Javadoc extraction, annotation parsing, call target analysis,
+and other Java/Kotlin/C# specific AST operations.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+
+# ---------------------------------------------------------------------------
+# Javadoc / docstring extraction
+# ---------------------------------------------------------------------------
+
+
+def extract_javadoc(node, lang: str) -> str | None:
+    """Extract the first meaningful line from JavaDoc/docstring above a node.
+
+    For Java/Kotlin/C#: extracts from /** ... */ block comments
+    For Python: extracts from triple-quoted docstrings
+
+    Returns the first non-annotation line (business description), or None.
+    """
+    if lang in ("java", "kotlin", "c_sharp"):
+        prev = node.prev_sibling
+        # Skip past annotations to find the comment
+        while prev and prev.type in ("marker_annotation", "annotation", "modifiers"):
+            prev = prev.prev_sibling
+
+        if prev and prev.type in ("comment", "block_comment", "line_comment"):
+            text = prev.text.decode("utf-8")
+            if "/**" in text:
+                lines = text.split("\n")
+                for line in lines:
+                    cleaned = line.strip().lstrip("/*").strip()
+                    if cleaned and not cleaned.startswith("@") and len(cleaned) > 3:
+                        return cleaned[:200]
+            elif text.strip().startswith("//"):
+                cleaned = text.strip().lstrip("/").strip()
+                if cleaned and len(cleaned) > 3:
+                    return cleaned[:200]
+
+    elif lang == "python":
+        body = node.child_by_field_name("body")
+        if body and body.children:
+            first = body.children[0]
+            if first.type == "expression_statement":
+                for child in first.children:
+                    if child.type == "string":
+                        text = child.text.decode("utf-8")
+                        text = text.strip("'\"").strip()
+                        first_line = text.split("\n")[0].strip()
+                        if first_line:
+                            return first_line[:200]
+
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Annotation extraction
+# ---------------------------------------------------------------------------
+
+
+def get_annotation_values(node, lang: str) -> list[dict]:
+    """Extract annotations WITH their attribute values (structured format).
+
+    Returns list of dicts: [{"name": "@KafkaListener", "attributes": {"topics": "order-events"}}]
+
+    Used for detecting implicit dependencies (Kafka topics, Feign clients, Redis keys).
+    """
+    if lang not in ("java", "kotlin", "c_sharp"):
+        return []
+
+    annotations = []
+    anno_nodes = []
+
+    for child in node.children:
+        if child.type in ("modifiers",):
+            for sub in child.children:
+                if sub.type in ("marker_annotation", "annotation"):
+                    anno_nodes.append(sub)
+        elif child.type in ("marker_annotation", "annotation"):
+            anno_nodes.append(child)
+
+    prev = node.prev_sibling
+    while prev and prev.type in ("marker_annotation", "annotation"):
+        anno_nodes.append(prev)
+        prev = prev.prev_sibling
+
+    for anno_node in anno_nodes:
+        text = anno_node.text.decode("utf-8")
+        name_match = re.match(r'@(\w+)', text)
+        if not name_match:
+            continue
+
+        anno_name = "@" + name_match.group(1)
+        attributes: dict[str, str] = {}
+
+        args_match = re.search(r'\((.+)\)', text, re.DOTALL)
+        if args_match:
+            args_text = args_match.group(1).strip()
+            # Key-value pairs: key = "value"
+            for kv in re.finditer(r'(\w+)\s*=\s*"([^"]*)"', args_text):
+                attributes[kv.group(1)] = kv.group(2)
+            # Array patterns: key = {values}
+            for kv in re.finditer(r'(\w+)\s*=\s*\{([^}]*)\}', args_text):
+                attributes[kv.group(1)] = kv.group(2).strip().strip('"')
+            # Single value: @Foo("value")
+            if not attributes:
+                single_match = re.match(r'^"([^"]*)"', args_text)
+                if single_match:
+                    attributes["value"] = single_match.group(1)
+
+        annotations.append({"name": anno_name, "attributes": attributes})
+
+    return annotations[:10]
+
+
+def parse_java_annotation(node, lang: str) -> str | dict:
+    """Parse a Java/Kotlin annotation node into structured format.
+
+    Simple annotations (no args): returns "@Service"
+    Annotations with args: returns {"name": "@Service", "attrs": {"value": "..."}}
+    """
+    if node.type == "marker_annotation":
+        return node.text.decode("utf-8")
+
+    name = None
+    attrs: dict[str, str] = {}
+
+    for child in node.children:
+        if child.type in ("identifier", "scoped_identifier"):
+            name = "@" + child.text.decode("utf-8")
+        elif child.type == "annotation_argument_list":
+            attrs = parse_annotation_args(child)
+
+    if not name:
+        return node.text.decode("utf-8")
+
+    if attrs:
+        return {"name": name, "attrs": attrs}
+    return name
+
+
+def parse_annotation_args(args_node) -> dict[str, str]:
+    """Parse annotation_argument_list into key-value dict.
+
+    Handles:
+      @Annotation("value")  → {"value": "\"value\""}
+      @Annotation(name = "x", url = "y")  → {"name": "\"x\"", "url": "\"y\""}
+    """
+    attrs: dict[str, str] = {}
+
+    for child in args_node.children:
+        if child.type == "element_value_pair":
+            key_node = child.child_by_field_name("key")
+            value_node = child.child_by_field_name("value")
+            if key_node and value_node:
+                key = key_node.text.decode("utf-8")
+                value = value_node.text.decode("utf-8")[:200]
+                attrs[key] = value
+        elif child.type in ("string_literal", "true", "false", "number_literal",
+                            "identifier", "field_access", "class_literal"):
+            attrs["value"] = child.text.decode("utf-8")[:200]
+        elif child.type == "element_value_array_initializer":
+            attrs["value"] = child.text.decode("utf-8")[:200]
+
+    return attrs
+
+
+# ---------------------------------------------------------------------------
+# Call target extraction
+# ---------------------------------------------------------------------------
+
+_CALL_NOISE = frozenset({
+    "this", "super", "System", "Math", "String", "Integer", "Long",
+    "Double", "Float", "Boolean", "Byte", "Short", "Character",
+    "Arrays", "Collections", "Objects", "Optional", "Stream",
+    "List", "Map", "Set", "HashMap", "ArrayList", "HashSet",
+    "StringBuilder", "StringBuffer", "Thread", "Class",
+    "Logger", "log", "logger", "LOG", "LOGGER",
+})
+
+
+def extract_call_targets(root, lang: str, content: str) -> list[str]:
+    """Extract class-level call targets from method bodies.
+
+    Identifies receiver objects in method calls
+    (e.g. 'orderService.createOrder()' → 'orderService').
+    Returns deduplicated list of likely dependency variable names.
+    """
+    from core.skeleton.parsers.treesitter import _walk_nodes, METHOD_NODES
+
+    if lang not in ("java", "kotlin", "c_sharp"):
+        return []
+
+    method_node_types = METHOD_NODES.get(lang, [])
+    all_receivers: set[str] = set()
+
+    for node in _walk_nodes(root, method_node_types):
+        body = node.text.decode("utf-8") if node.text else ""
+        for m in re.finditer(r'\b(\w+)\.(\w+)\s*\(', body):
+            receiver = m.group(1)
+            if receiver in _CALL_NOISE:
+                continue
+            if receiver[0].isupper() and receiver not in _CALL_NOISE:
+                all_receivers.add(receiver)
+            elif receiver[0].islower() and len(receiver) > 2:
+                all_receivers.add(receiver)
+
+    return sorted(all_receivers)[:50]
+
+
+def extract_method_calls(node, lang: str) -> list[str]:
+    """Extract call targets from a single method body.
+
+    Returns list of receiver names (e.g. ['orderService', 'paymentService']).
+    """
+    if lang not in ("java", "kotlin", "c_sharp"):
+        return []
+
+    body = node.text.decode("utf-8") if node.text else ""
+    if not body:
+        return []
+
+    receivers: set[str] = set()
+    for m in re.finditer(r'\b(\w+)\.(\w+)\s*\(', body):
+        receiver = m.group(1)
+        if receiver in _CALL_NOISE or len(receiver) <= 2:
+            continue
+        if receiver[0].islower() or (receiver[0].isupper() and receiver not in _CALL_NOISE):
+            receivers.add(receiver)
+
+    return sorted(receivers)[:20]
+
+
+# ---------------------------------------------------------------------------
+# Inheritance extraction
+# ---------------------------------------------------------------------------
+
+
+def get_extends(node, lang: str) -> str | None:
+    """Extract superclass."""
+    sc = node.child_by_field_name("superclass")
+    if sc:
+        return sc.text.decode("utf-8")
+    for child in node.children:
+        if child.type == "superclass":
+            return child.text.decode("utf-8").replace("extends ", "")
+    return None
+
+
+def get_implements(node, lang: str) -> list[str] | None:
+    """Extract implemented interfaces."""
+    for child in node.children:
+        if child.type in ("super_interfaces", "interfaces"):
+            text = child.text.decode("utf-8")
+            text = text.replace("implements ", "")
+            return [i.strip() for i in text.split(",")]
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Complexity and constants
+# ---------------------------------------------------------------------------
+
+
+def count_branches(node) -> int:
+    """Count branching nodes for complexity analysis."""
+    from core.skeleton.parsers.treesitter import _walk_nodes, BRANCH_TYPES
+    count = 0
+    for _ in _walk_nodes(node, list(BRANCH_TYPES)):
+        count += 1
+    return count
+
+
+def extract_constants(node, content: str) -> list[str]:
+    """Extract constant references from method body (UPPER_CASE identifiers)."""
+    body_text = node.text.decode("utf-8") if node.text else ""
+    constants = set(re.findall(r'\b([A-Z][A-Z0-9_]{2,})\b', body_text))
+    noise = {"NULL", "TRUE", "FALSE", "NONE", "THIS", "SELF", "GET", "SET", "PUT", "POST", "DELETE"}
+    return sorted(constants - noise)

core/skeleton/parsers/treesitter_multi.py

@@ -0,0 +1,289 @@
+"""Multi-language tree-sitter specializations.
+
+Handles fallback parsing, document-based method extraction, field extraction,
+and extraction orchestration for classes/methods/fields/imports.
+"""
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Any
+
+
+# ---------------------------------------------------------------------------
+# Fallback regex-based parsing
+# ---------------------------------------------------------------------------
+
+def fallback_parse(content: str, filepath: str) -> dict[str, Any] | None:
+    """Fallback to simple regex parsing when tree-sitter can't handle the file."""
+    lines = content.splitlines()
+    total_lines = len(lines)
+    methods: list[dict[str, Any]] = []
+    classes: list[dict[str, Any]] = []
+
+    func_re = re.compile(
+        r'^\s*(?:(?:public|private|protected|static|async|export|def|func|fn|fun)\s+)*'
+        r'(?:function\s+)?(\w+)\s*\('
+    )
+    class_re = re.compile(
+        r'^\s*(?:(?:public|abstract|final|export)\s+)*'
+        r'(?:class|struct|interface|enum|trait|object|type)\s+(\w+)'
+    )
+    branch_re = re.compile(
+        r'^\s*(?:if|else|elif|switch|case|match|for|while|catch|except)\b'
+    )
+
+    func_starts: list[tuple[str, int]] = []
+    for i, line in enumerate(lines):
+        m = func_re.match(line)
+        if m:
+            func_starts.append((m.group(1), i))
+        m = class_re.match(line)
+        if m:
+            classes.append({"name": m.group(1)})
+
+    for idx, (name, start) in enumerate(func_starts):
+        end = func_starts[idx + 1][1] if idx + 1 < len(func_starts) else total_lines
+        line_count = end - start
+        branches = sum(1 for ln in lines[start:end] if branch_re.match(ln))
+
+        if line_count > 50 or branches > 10:
+            complexity = "high"
+        elif line_count > 25 or branches > 5:
+            complexity = "medium"
+        else:
+            complexity = "low"
+
+        methods.append({
+            "name": name, "access": "public", "line_count": line_count,
+            "complexity": complexity, "branches": branches,
+        })
+
+    if not methods and not classes:
+        return None
+    return {"file": filepath, "total_lines": total_lines, "methods": methods, "classes": classes}
+
+
+# ---------------------------------------------------------------------------
+# Document-based method extraction (for coverage checks)
+# ---------------------------------------------------------------------------
+
+def extract_methods_from_doc(doc_path: str | Path) -> tuple[set[str], set[str]]:
+    """Extract method names and class names from documentation files.
+
+    Returns (methods, classes) sets.
+    """
+    try:
+        content = Path(doc_path).read_text(encoding="utf-8")
+    except UnicodeDecodeError:
+        try:
+            content = Path(doc_path).read_text(encoding="gbk")
+        except Exception:
+            return set(), set()
+
+    methods: set[str] = set()
+    classes: set[str] = set()
+
+    for m in re.finditer(r'`(\w+(?:\.\w+)?)`', content):
+        token = m.group(1)
+        if "." in token:
+            parts = token.split(".")
+            classes.add(parts[0])
+            methods.add(parts[-1])
+        elif token[0].isupper():
+            classes.add(token)
+        else:
+            methods.add(token)
+
+    for m in re.finditer(r'^#{2,4}\s+.*?(\w+)\s*$', content, re.MULTILINE):
+        token = m.group(1)
+        if token[0].isupper():
+            classes.add(token)
+        else:
+            methods.add(token)
+
+    return methods, classes
+
+
+# ---------------------------------------------------------------------------
+# Language-specific field extraction
+# ---------------------------------------------------------------------------
+
+def get_field_name(node, lang: str) -> str | None:
+    """Extract field name from a field declaration."""
+    name_node = node.child_by_field_name("declarator") or node.child_by_field_name("name")
+    if name_node:
+        inner = name_node.child_by_field_name("name")
+        if inner:
+            return inner.text.decode("utf-8")
+        return name_node.text.decode("utf-8").split("=")[0].strip()
+    return None
+
+
+def get_field_type(node, lang: str) -> str | None:
+    """Extract field type from a field declaration."""
+    type_node = node.child_by_field_name("type")
+    if type_node:
+        return type_node.text.decode("utf-8")[:100]
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Extraction orchestration (classes, methods, fields, imports)
+# ---------------------------------------------------------------------------
+
+def extract_classes(root, lang: str) -> list[dict]:
+    """Extract class/interface/enum declarations with full metadata."""
+    from core.skeleton.parsers.treesitter import (
+        _walk_nodes, _get_name, _classify_type, _get_annotations, CLASS_NODES,
+    )
+    from core.skeleton.parsers.treesitter_java import (
+        extract_javadoc, get_annotation_values, get_extends, get_implements,
+    )
+    node_types = CLASS_NODES.get(lang, [])
+    classes = []
+    for node in _walk_nodes(root, node_types):
+        name = _get_name(node, lang)
+        if not name:
+            continue
+        cls: dict[str, Any] = {"name": name, "type": _classify_type(node, lang)}
+        annotations = _get_annotations(node, lang)
+        if annotations:
+            cls["annotations"] = annotations
+        anno_values = get_annotation_values(node, lang)
+        if anno_values:
+            cls["annotation_values"] = anno_values
+        extends = get_extends(node, lang)
+        if extends:
+            cls["extends"] = extends
+        implements = get_implements(node, lang)
+        if implements:
+            cls["implements"] = implements
+        javadoc = extract_javadoc(node, lang)
+        if javadoc:
+            cls["doc"] = javadoc
+        classes.append(cls)
+    return classes
+
+
+def extract_methods(root, lang: str, content: str) -> list[dict]:
+    """Extract method declarations with complexity analysis."""
+    from core.skeleton.parsers.treesitter import (
+        _walk_nodes, _get_name, _get_access, _get_params,
+        _get_return_type, _get_annotations, METHOD_NODES,
+    )
+    from core.skeleton.parsers.treesitter_java import (
+        extract_javadoc, get_annotation_values, extract_constants,
+        extract_method_calls, count_branches,
+    )
+    node_types = METHOD_NODES.get(lang, [])
+    methods = []
+    for node in _walk_nodes(root, node_types):
+        name = _get_name(node, lang)
+        if not name:
+            continue
+        line_count = node.end_point[0] - node.start_point[0] + 1
+        branches = count_branches(node)
+        if line_count > 50 or branches > 10:
+            complexity = "high"
+        elif line_count > 25 or branches > 5:
+            complexity = "medium"
+        else:
+            complexity = "low"
+        method: dict[str, Any] = {
+            "name": name, "access": _get_access(node, lang),
+            "line_count": line_count, "complexity": complexity, "branches": branches,
+        }
+        params = _get_params(node, lang)
+        if params is not None:
+            method["params"] = params
+        return_type = _get_return_type(node, lang)
+        if return_type:
+            method["return_type"] = return_type
+        annotations = _get_annotations(node, lang)
+        if annotations:
+            method["annotations"] = annotations
+        anno_values = get_annotation_values(node, lang)
+        if anno_values:
+            method["annotation_values"] = anno_values
+        javadoc = extract_javadoc(node, lang)
+        if javadoc:
+            method["doc"] = javadoc
+        constants = extract_constants(node, content)
+        if constants:
+            method["constants"] = constants[:30]
+        method_calls = extract_method_calls(node, lang)
+        if method_calls:
+            method["calls"] = method_calls
+        methods.append(method)
+    return methods
+
+
+def extract_fields(root, lang: str) -> list[dict]:
+    """Extract class fields/properties."""
+    from core.skeleton.parsers.treesitter import _walk_nodes, _get_annotations
+    field_types = {
+        "java": ["field_declaration"], "kotlin": ["property_declaration"],
+        "typescript": ["public_field_definition", "property_declaration"],
+        "c_sharp": ["field_declaration", "property_declaration"], "python": [],
+    }
+    node_types = field_types.get(lang, [])
+    if not node_types:
+        return []
+    fields = []
+    for node in _walk_nodes(root, node_types):
+        name = get_field_name(node, lang)
+        if name:
+            field: dict[str, Any] = {"name": name}
+            ftype = get_field_type(node, lang)
+            if ftype:
+                field["type"] = ftype
+            annotations = _get_annotations(node, lang)
+            if annotations:
+                field["annotations"] = annotations
+            fields.append(field)
+    return fields
+
+
+def extract_imports(root, lang: str) -> list[str]:
+    """Extract import statements for all supported languages."""
+    from core.skeleton.parsers.treesitter import _walk_nodes
+    import_node_types = {
+        "java": ["import_declaration"], "python": ["import_statement", "import_from_statement"],
+        "go": ["import_declaration", "import_spec"], "typescript": ["import_statement"],
+        "tsx": ["import_statement"], "javascript": ["import_statement"],
+        "kotlin": ["import_header"], "c_sharp": ["using_directive"],
+    }
+    node_types = import_node_types.get(lang, [])
+    if not node_types:
+        return []
+    imports = []
+    for node in _walk_nodes(root, node_types):
+        text = node.text.decode("utf-8").strip()
+        if lang == "java":
+            m = re.search(r'import\s+(?:static\s+)?([\w.]+)', text)
+            if m and not m.group(1).startswith(("java.", "javax.", "sun.", "jdk.")):
+                imports.append(m.group(1))
+        elif lang == "python":
+            m = re.search(r'from\s+([\w.]+)\s+import', text) or re.search(r'import\s+([\w.]+)', text)
+            if m:
+                imports.append(m.group(1))
+        elif lang == "go":
+            m = re.search(r'"([^"]+)"', text)
+            if m:
+                imports.append(m.group(1))
+        elif lang in ("typescript", "tsx", "javascript"):
+            m = re.search(r'''from\s+['"]([^'"]+)['"]''', text)
+            if not m:
+                m = re.search(r'''import\s+['"]([^'"]+)['"]''', text)
+            if m:
+                imports.append(m.group(1))
+        elif lang == "kotlin":
+            m = re.search(r'import\s+([\w.]+)', text)
+            if m and not m.group(1).startswith(("java.", "javax.", "kotlin.")):
+                imports.append(m.group(1))
+        elif lang == "c_sharp":
+            m = re.search(r'using\s+(?:static\s+)?([\w.]+)', text)
+            if m and not m.group(1).startswith("System."):
+                imports.append(m.group(1))
+    return imports