source-kb 0.2.2__py3-none-any.whl

core/skeleton/parsers/__init__.py

@@ -0,0 +1,71 @@
+"""Skeleton parser registry.
+
+Parsers are registered by name and selected based on preset configuration.
+The parser chain tries each parser in priority order until one succeeds.
+
+Usage:
+    from core.skeleton.parsers import get_parser_chain, register_parser
+
+    chain = get_parser_chain(preset)
+    for parser in chain:
+        if parser.can_parse(repo_path, preset):
+            entries = parser.parse(repo_path, preset)
+            break
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any
+
+from core.interfaces import SkeletonParser
+
+logger = logging.getLogger(__name__)
+
+_REGISTRY: dict[str, type[SkeletonParser]] = {}
+
+
+def register_parser(cls: type[SkeletonParser]) -> type[SkeletonParser]:
+    """Decorator to register a parser class by its `name` attribute."""
+    _REGISTRY[cls.name] = cls
+    return cls
+
+
+def get_parser_chain(preset: dict[str, Any]) -> list[SkeletonParser]:
+    """Return parser instances in preset-declared priority order.
+
+    Falls back to ["treesitter", "regex"] if preset doesn't declare parsers.
+    Skips unknown parser names with a warning.
+    """
+    names = preset.get("parsers", ["treesitter", "regex"])
+    chain: list[SkeletonParser] = []
+    for name in names:
+        cls = _REGISTRY.get(name)
+        if cls is None:
+            logger.warning("Unknown parser '%s' in preset config, skipping", name)
+            continue
+        chain.append(cls())
+    return chain
+
+
+def available_parsers() -> list[str]:
+    """Return names of all registered parsers."""
+    return sorted(_REGISTRY.keys())
+
+
+def get_parser(name: str) -> SkeletonParser:
+    """Get a specific parser by name. Raises KeyError if not found."""
+    cls = _REGISTRY.get(name)
+    if cls is None:
+        raise KeyError(f"Parser '{name}' not registered. Available: {available_parsers()}")
+    return cls()
+
+
+# ---------------------------------------------------------------------------
+# Auto-import parser submodules to trigger @register_parser decorators.
+# Without these imports, the decorator never executes and _REGISTRY stays empty.
+# ---------------------------------------------------------------------------
+from core.skeleton.parsers import treesitter  # noqa: F401, E402
+from core.skeleton.parsers import jqassistant  # noqa: F401, E402
+from core.skeleton.parsers import regex  # noqa: F401, E402

core/skeleton/parsers/jqassistant.py

@@ -0,0 +1,300 @@
+"""jQAssistant-based skeleton parser — Java bytecode analysis.
+
+Uses jQAssistant + embedded Neo4j to extract precise class/method/field/annotation
+data from compiled .class files. Falls back gracefully if unavailable.
+
+Requires: jQAssistant CLI on PATH (or JQASSISTANT_HOME), Java 11+,
+compiled .class files (target/classes or build/classes).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import re
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+from typing import Any
+
+from core.interfaces import SkeletonParser
+from core.skeleton.parsers import register_parser
+from core.skeleton.parsers.jqassistant_cypher import (
+    CYPHER_ANNOTATIONS, CYPHER_CALL_GRAPH, CYPHER_CLASSES,
+    CYPHER_FIELDS, CYPHER_METHODS,
+    apply_annotations, apply_fields, apply_methods,
+    build_call_graph_edges, build_class_entries, parse_cypher_output,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def is_available() -> bool:
+    """Check if jQAssistant CLI is available."""
+    return _find_jqa_command() is not None
+
+
+def _find_jqa_command() -> str | None:
+    """Locate jQAssistant CLI executable."""
+    for cmd in ("jqassistant", "jqassistant.cmd", "jqassistant.bat"):
+        if shutil.which(cmd):
+            return cmd
+    jqa_home = os.environ.get("JQASSISTANT_HOME")
+    if jqa_home:
+        for suffix in ("bin/jqassistant", "bin/jqassistant.cmd", "bin/jqassistant.bat"):
+            full = os.path.join(jqa_home, suffix)
+            if os.path.isfile(full):
+                return full
+    return None
+
+
+@register_parser
+class JQAssistantParser(SkeletonParser):
+    """jQAssistant parser. Registered as 'jqassistant'. Java-only, highest priority."""
+
+    name = "jqassistant"
+    priority = 100
+
+    def can_parse(self, repo_path: Path, preset: dict[str, Any]) -> bool:
+        """Available only if jQAssistant CLI exists AND compiled classes are present."""
+        if not is_available():
+            return False
+        candidates = [
+            repo_path / "target" / "classes",
+            repo_path / "build" / "classes" / "java" / "main",
+        ]
+        # Multi-module: check sub-modules
+        for sub in repo_path.iterdir():
+            if sub.is_dir() and (sub / "pom.xml").exists():
+                candidates.append(sub / "target" / "classes")
+        return any(d.is_dir() and any(d.rglob("*.class")) for d in candidates)
+
+    def parse(self, repo_path: Path, preset: dict[str, Any], **kwargs: Any) -> list[dict[str, Any]]:
+        """Scan compiled classes and extract skeleton entries via Cypher queries."""
+        jqa_cmd = _find_jqa_command()
+        if not jqa_cmd:
+            return []
+
+        classes_dir = self._find_classes_dir(repo_path)
+        if not classes_dir:
+            return []
+
+        store_dir = tempfile.mkdtemp(prefix="jqa-")
+        try:
+            if not self._run_scan(jqa_cmd, store_dir, classes_dir):
+                return []
+            return self._query_all_skeleton(jqa_cmd, store_dir, repo_path)
+        finally:
+            shutil.rmtree(store_dir, ignore_errors=True)
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _find_classes_dir(self, repo_path: Path) -> Path | None:
+        """Find the compiled classes directory."""
+        candidates = [
+            repo_path / "target" / "classes",
+            repo_path / "build" / "classes" / "java" / "main",
+        ]
+        for d in candidates:
+            if d.is_dir() and any(d.rglob("*.class")):
+                return d
+        # Multi-module: find first sub-module with classes
+        for sub in sorted(repo_path.iterdir()):
+            if sub.is_dir() and (sub / "target" / "classes").is_dir():
+                return sub / "target" / "classes"
+        return None
+
+    def _run_scan(self, jqa_cmd: str, store_dir: str, classes_dir: Path) -> bool:
+        """Run jQAssistant scan on classes directory."""
+        try:
+            result = subprocess.run(
+                [jqa_cmd, "scan", "-f", str(classes_dir), "-s", store_dir],
+                capture_output=True, encoding="utf-8", errors="replace", timeout=300,
+            )
+            return result.returncode == 0
+        except (subprocess.TimeoutExpired, OSError) as e:
+            logger.warning("jQAssistant scan failed: %s", e)
+            return False
+
+    def _query_all_skeleton(
+        self, jqa_cmd: str, store_dir: str, repo_path: Path
+    ) -> list[dict[str, Any]]:
+        """Query Neo4j store for all skeleton data and assemble entries."""
+        classes_data = self._run_cypher(jqa_cmd, store_dir, repo_path, CYPHER_CLASSES)
+        methods_data = self._run_cypher(jqa_cmd, store_dir, repo_path, CYPHER_METHODS)
+        fields_data = self._run_cypher(jqa_cmd, store_dir, repo_path, CYPHER_FIELDS)
+        annotations_data = self._run_cypher(jqa_cmd, store_dir, repo_path, CYPHER_ANNOTATIONS)
+
+        cache = build_class_entries(classes_data)
+        apply_annotations(cache, annotations_data)
+        apply_methods(cache, methods_data)
+        apply_fields(cache, fields_data)
+
+        return list(cache.values())
+
+    def _run_cypher(
+        self, jqa_cmd: str, store_dir: str, repo_path: Path, query: str
+    ) -> list[dict]:
+        """Execute a Cypher query against the jQAssistant store."""
+        with tempfile.NamedTemporaryFile(
+            mode="w", suffix=".cypher", delete=False
+        ) as f:
+            f.write(query)
+            query_file = f.name
+
+        try:
+            cmd = [
+                jqa_cmd, "report",
+                "-s", store_dir,
+                "-concepts", query_file,
+                "-reportFormat", "json",
+            ]
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                encoding="utf-8",
+                errors="replace",
+                timeout=120,
+                cwd=str(repo_path),
+            )
+            if result.returncode != 0:
+                logger.warning("Cypher query failed: %s", result.stderr[:500])
+                return []
+
+            try:
+                data = json.loads(result.stdout)
+                return data if isinstance(data, list) else data.get("rows", [])
+            except json.JSONDecodeError:
+                return parse_cypher_output(result.stdout)
+        except (subprocess.TimeoutExpired, OSError) as e:
+            logger.warning("Cypher query error: %s", e)
+            return []
+        finally:
+            os.unlink(query_file)
+
+
+# ------------------------------------------------------------------
+# Source-only fallback parsing (when jQAssistant scan is unavailable)
+# ------------------------------------------------------------------
+
+_CLASS_PATTERN = re.compile(
+    r'(?:public|protected|private)?\s*(?:abstract|final|static)?\s*'
+    r'(class|interface|enum|record)\s+(\w+)'
+)
+
+_METHOD_PATTERN = re.compile(
+    r'(?:public|protected|private|static|\s)+[\w<>\[\],\s]+\s+(\w+)\s*\('
+)
+
+_CONSTANT_PATTERN = re.compile(
+    r'(?:public|private|protected)?\s*static\s+final\s+'
+    r'(\w+)\s+([A-Z_][A-Z0-9_]*)\s*=\s*(.+?)\s*;'
+)
+
+
+def parse_source_only(content: str, filepath: str) -> dict[str, Any] | None:
+    """Fallback: extract basic skeleton from source when jQAssistant data unavailable."""
+    lines = content.split("\n")
+    if not lines:
+        return None
+    entry: dict[str, Any] = {
+        "file": filepath, "total_lines": len(lines),
+        "classes": [], "methods": [], "fields": [],
+    }
+
+    for line in lines:
+        m = _CLASS_PATTERN.search(line)
+        if m:
+            entry["classes"].append({
+                "name": m.group(2),
+                "type": m.group(1),
+                "annotations": [],
+            })
+
+    method_metrics = _extract_method_line_counts(content)
+    for name, metrics in method_metrics.items():
+        entry["methods"].append({
+            "name": name, "access": "public", "static": False,
+            "return_type": "", "params": "", "annotations": [],
+            "line_count": metrics["line_count"],
+            "complexity": metrics["complexity"],
+            "branches": metrics["branches"], "constants": [],
+        })
+
+    return entry if entry["classes"] or entry["methods"] else None
+
+
+def enrich_with_source_metrics(entry: dict[str, Any], content: str) -> None:
+    """Enrich jQAssistant entry with source-level metrics (line counts, complexity)."""
+    entry["total_lines"] = len(content.split("\n"))
+    method_lines = _extract_method_line_counts(content)
+    for method in entry.get("methods", []):
+        name = method.get("name", "")
+        if name in method_lines:
+            method["line_count"] = method_lines[name]["line_count"]
+            method["complexity"] = method_lines[name]["complexity"]
+            method["branches"] = method_lines[name]["branches"]
+    constants = _extract_constants(content)
+    if constants:
+        entry.setdefault("constants", []).extend(constants)
+
+
+def _extract_method_line_counts(content: str) -> dict[str, dict]:
+    """Extract method line counts and complexity from source."""
+    results: dict[str, dict] = {}
+    lines = content.split("\n")
+    branch_keywords = (
+        "if ", "if(", "else ", "for ", "for(", "while ", "while(",
+        "switch ", "switch(", "catch ", "catch(", "case ",
+    )
+
+    i = 0
+    while i < len(lines):
+        match = _METHOD_PATTERN.search(lines[i])
+        if match:
+            name = match.group(1)
+            start = i
+            brace_count = 0
+            found_open = False
+            branches = 0
+
+            for j in range(i, len(lines)):
+                line = lines[j]
+                brace_count += line.count("{") - line.count("}")
+                if "{" in line:
+                    found_open = True
+                if found_open and brace_count <= 0:
+                    line_count = j - start + 1
+                    complexity = (
+                        "low" if branches <= 2
+                        else ("medium" if branches <= 5 else "high")
+                    )
+                    results[name] = {
+                        "line_count": line_count,
+                        "complexity": complexity,
+                        "branches": branches,
+                    }
+                    i = j
+                    break
+
+                if any(kw in line for kw in branch_keywords):
+                    branches += 1
+        i += 1
+
+    return results
+
+
+def _extract_constants(content: str) -> list[dict[str, str]]:
+    """Extract static final constants from source."""
+    constants: list[dict[str, str]] = []
+    for match in _CONSTANT_PATTERN.finditer(content):
+        constants.append({
+            "name": match.group(2),
+            "type": match.group(1),
+            "value": match.group(3)[:100],
+        })
+    return constants

core/skeleton/parsers/jqassistant_cypher.py

@@ -0,0 +1,225 @@
+"""Cypher query templates and data transformation for jQAssistant parser.
+
+Contains all Cypher queries used to extract skeleton data from jQAssistant's
+embedded Neo4j graph database, plus helper functions that transform raw query
+results into the skeleton entry format expected by the parser chain.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+# ============================================================
+# Cypher Query Templates
+# ============================================================
+
+CYPHER_CLASSES = """
+MATCH (t:Type:Class)
+WHERE NOT t:Inner
+RETURN t.fqn AS fqn, t.name AS name, 'class' AS type,
+       COALESCE(t.superClass, '') AS superClass,
+       [(t)-[:IMPLEMENTS]->(i) | i.fqn] AS interfaces
+UNION ALL
+MATCH (t:Type:Interface)
+RETURN t.fqn AS fqn, t.name AS name, 'interface' AS type,
+       '' AS superClass, [] AS interfaces
+UNION ALL
+MATCH (t:Type:Enum)
+RETURN t.fqn AS fqn, t.name AS name, 'enum' AS type,
+       '' AS superClass, [] AS interfaces
+"""
+
+CYPHER_METHODS = """
+MATCH (t:Type)-[:DECLARES]->(m:Method)
+WHERE NOT m.name STARTS WITH '<' AND NOT t:Inner
+RETURN t.fqn AS classFqn, m.name AS name,
+       m.visibility AS visibility,
+       m.static AS static,
+       COALESCE(m.returnType, 'void') AS returnType,
+       [(m)-[:HAS]->(p:Parameter) | p.type + ' ' + p.name] AS params,
+       [(m)-[:ANNOTATED_BY]->(a) | '@' + last(split(a.fqn, '.'))] AS annotations
+"""
+
+CYPHER_FIELDS = """
+MATCH (t:Type)-[:DECLARES]->(f:Field)
+WHERE NOT t:Inner
+RETURN t.fqn AS classFqn, f.name AS name,
+       COALESCE(f.type, '') AS type,
+       [(f)-[:ANNOTATED_BY]->(a) | '@' + last(split(a.fqn, '.'))] AS annotations
+"""
+
+CYPHER_ANNOTATIONS = """
+MATCH (t:Type)-[:ANNOTATED_BY]->(a:Annotation)-[:OF_TYPE]->(at:Type)
+WHERE NOT t:Inner
+RETURN t.fqn AS classFqn, t.name AS className,
+       at.fqn AS annotationType,
+       'class' AS target,
+       '' AS methodName,
+       COALESCE(a.value, '') AS value
+UNION ALL
+MATCH (t:Type)-[:DECLARES]->(m:Method)-[:ANNOTATED_BY]->(a:Annotation)-[:OF_TYPE]->(at:Type)
+WHERE NOT t:Inner AND NOT m.name STARTS WITH '<'
+RETURN t.fqn AS classFqn, t.name AS className,
+       at.fqn AS annotationType,
+       'method' AS target,
+       m.name AS methodName,
+       COALESCE(a.value, '') AS value
+"""
+
+CYPHER_CALL_GRAPH = """
+MATCH (caller:Type)-[:DECLARES]->(m:Method)-[:INVOKES]->(callee_m:Method)<-[:DECLARES]-(callee:Type)
+WHERE NOT caller:Inner AND NOT callee:Inner
+  AND caller <> callee
+  AND NOT m.name STARTS WITH '<'
+  AND NOT callee_m.name STARTS WITH '<'
+RETURN caller.fqn AS callerClass, callee.fqn AS calleeClass,
+       callee_m.name AS calleeMethod
+ORDER BY callerClass, calleeClass
+"""
+
+# All queries grouped for convenience
+CYPHER_QUERIES = {
+    "classes": CYPHER_CLASSES,
+    "methods": CYPHER_METHODS,
+    "fields": CYPHER_FIELDS,
+    "annotations": CYPHER_ANNOTATIONS,
+    "call_graph": CYPHER_CALL_GRAPH,
+}
+
+
+# ============================================================
+# Data Transformation Functions
+# ============================================================
+
+
+def fqn_to_source_path(fqn: str) -> str:
+    """Convert Java FQN to source file path.
+
+    com.example.service.MyService → src/main/java/com/example/service/MyService.java
+    """
+    if not fqn:
+        return ""
+    parts = fqn.replace(".", "/")
+    if "$" in parts:
+        parts = parts.split("$")[0]
+    return f"src/main/java/{parts}.java"
+
+
+def build_class_entries(classes_data: list[dict]) -> dict[str, dict[str, Any]]:
+    """Transform raw class query results into skeleton cache entries.
+
+    Returns a dict keyed by source file path.
+    """
+    cache: dict[str, dict[str, Any]] = {}
+    for row in classes_data:
+        fqn = row.get("fqn", "")
+        source_file = fqn_to_source_path(fqn)
+        if not source_file:
+            continue
+        if source_file not in cache:
+            cache[source_file] = {
+                "file": source_file,
+                "total_lines": 0,
+                "classes": [],
+                "methods": [],
+                "fields": [],
+            }
+        cache[source_file]["classes"].append({
+            "name": row.get("name", ""),
+            "type": row.get("type", "class").lower(),
+            "annotations": [],
+            "extends": row.get("superClass", ""),
+            "implements": row.get("interfaces", []),
+        })
+    return cache
+
+
+def apply_annotations(cache: dict[str, dict], annotations_data: list[dict]) -> None:
+    """Apply annotation data to classes and methods in the cache (mutates in place)."""
+    for row in annotations_data:
+        source_file = fqn_to_source_path(row.get("classFqn", ""))
+        if source_file not in cache:
+            continue
+        anno_name = "@" + row.get("annotationType", "").split(".")[-1]
+        anno_value = row.get("value", "")
+        anno_str = f"{anno_name}({anno_value})" if anno_value else anno_name
+
+        target = row.get("target", "class")
+        if target == "class":
+            for cls in cache[source_file]["classes"]:
+                if cls["name"] == row.get("className", ""):
+                    cls["annotations"].append(anno_str)
+                    break
+        elif target == "method":
+            for m in cache[source_file]["methods"]:
+                if m["name"] == row.get("methodName", ""):
+                    m.setdefault("annotations", []).append(anno_str)
+                    break
+
+
+def apply_methods(cache: dict[str, dict], methods_data: list[dict]) -> None:
+    """Apply method data to the cache (mutates in place)."""
+    for row in methods_data:
+        source_file = fqn_to_source_path(row.get("classFqn", ""))
+        if source_file not in cache:
+            continue
+        cache[source_file]["methods"].append({
+            "name": row.get("name", ""),
+            "access": row.get("visibility", "public").lower(),
+            "static": row.get("static", False),
+            "return_type": row.get("returnType", "void"),
+            "params": row.get("params", ""),
+            "annotations": row.get("annotations", []),
+            "line_count": 0,
+            "complexity": "low",
+            "branches": 0,
+            "constants": [],
+        })
+
+
+def apply_fields(cache: dict[str, dict], fields_data: list[dict]) -> None:
+    """Apply field data to the cache (mutates in place)."""
+    for row in fields_data:
+        source_file = fqn_to_source_path(row.get("classFqn", ""))
+        if source_file not in cache:
+            continue
+        cache[source_file]["fields"].append({
+            "name": row.get("name", ""),
+            "type": row.get("type", "").split(".")[-1],
+            "annotations": row.get("annotations", []),
+        })
+
+
+def build_call_graph_edges(results: list[dict]) -> list[dict[str, Any]]:
+    """Aggregate call graph results into caller-callee edge list.
+
+    Returns: [{"caller": "OrderService", "callee": "PaymentService", "methods": ["pay"]}]
+    """
+    edges: dict[tuple[str, str], list[str]] = {}
+    for row in results:
+        caller = row.get("callerClass", "").split(".")[-1]
+        callee = row.get("calleeClass", "").split(".")[-1]
+        method = row.get("calleeMethod", "")
+        if caller and callee and caller != callee:
+            key = (caller, callee)
+            edges.setdefault(key, []).append(method)
+
+    return [
+        {"caller": k[0], "callee": k[1], "methods": sorted(set(v))[:10]}
+        for k, v in edges.items()
+    ]
+
+
+def parse_cypher_output(output: str) -> list[dict]:
+    """Parse non-JSON Cypher output (tab-separated or similar)."""
+    rows: list[dict] = []
+    lines = output.strip().split("\n")
+    if len(lines) < 2:
+        return rows
+
+    headers = [h.strip() for h in lines[0].split("\t")]
+    for line in lines[1:]:
+        values = [v.strip() for v in line.split("\t")]
+        if len(values) == len(headers):
+            rows.append(dict(zip(headers, values)))
+    return rows