codeboarding 0.9.0__py3-none-any.whl

static_analyzer/lsp_client/typescript_client.py

@@ -0,0 +1,235 @@
+import logging
+import os
+import time
+
+from .client import LSPClient
+
+logger = logging.getLogger(__name__)
+
+
+class TypeScriptClient(LSPClient):
+    """
+    TypeScript/JavaScript-specific Language Server Protocol client.
+    Extends the base LSPClient with TypeScript-specific functionality.
+    """
+
+    def handle_notification(self, method: str, params: dict):
+        """
+        Handle notifications from the TypeScript language server.
+
+        TypeScript language server notifications are not needed for our analysis,
+        so this method intentionally does nothing. The base implementation is sufficient.
+
+        Args:
+            method: The LSP notification method name
+            params: The notification parameters
+        """
+        # TypeScript LSP server notifications are not needed for static analysis
+        # The server handles project loading internally without requiring client tracking
+        pass
+
+    def start(self):
+        """Starts the language server with dependency check."""
+        # Check and install dependencies if needed
+        self._ensure_dependencies()
+
+        # Call parent start method
+        super().start()
+
+    def _ensure_dependencies(self):
+        """Check if node_modules exists and log an error if they don't."""
+        node_modules_path = self.project_path / "node_modules"
+
+        if node_modules_path.exists():
+            logger.info(f"node_modules found at: {node_modules_path}")
+            return
+
+        logger.warning(f"node_modules not found in {self.project_path}")
+
+        # Check if package.json exists
+        package_json = self.project_path / "package.json"
+        if not package_json.exists():
+            logger.warning(f"package.json not found in {self.project_path}.")
+            return
+
+    def _initialize(self):
+        """Performs the LSP initialization handshake."""
+        logger.info(f"Initializing connection for {self.language_id}...")
+        params = {
+            "processId": os.getpid(),
+            "rootUri": self.project_path.as_uri(),
+            "capabilities": {
+                "textDocument": {
+                    "callHierarchy": {"dynamicRegistration": True},
+                    "documentSymbol": {"hierarchicalDocumentSymbolSupport": True},
+                    "typeHierarchy": {"dynamicRegistration": True},
+                    "references": {"dynamicRegistration": True},
+                    "semanticTokens": {"dynamicRegistration": True},
+                },
+                "workspace": {
+                    "configuration": True,
+                    "workspaceFolders": True,
+                    "didChangeConfiguration": {"dynamicRegistration": True},
+                },
+            },
+            "workspace": {"applyEdit": True, "workspaceEdit": {"documentChanges": True}},
+        }
+
+        # Allow subclasses to customize initialization parameters
+        params = self._customize_initialization_params(params)
+
+        init_id = self._send_request("initialize", params)
+        # Use longer timeout for initialization as it may involve full workspace indexing
+        response = self._wait_for_response(init_id, timeout=360)
+
+        if "error" in response:
+            raise RuntimeError(f"Initialization failed: {response['error']}")
+
+        logger.info("Initialization successful.")
+        self._send_notification("initialized", {})
+
+        # Allow subclasses to perform post-initialization setup
+        self._configure_typescript_workspace()
+
+    def _customize_initialization_params(self, params: dict) -> dict:
+        """Add TypeScript-specific initialization parameters."""
+        params["workspaceFolders"] = [{"uri": self.project_path.as_uri(), "name": self.project_path.name}]
+
+        params["initializationOptions"] = {
+            "preferences": {"includeCompletionsForModuleExports": True, "includeCompletionsWithSnippetText": True},
+            "tsserver": {"logVerbosity": "off"},  # Reduce noise in logs
+        }
+
+        return params
+
+    def _configure_typescript_workspace(self):
+        """Send TypeScript-specific workspace configuration after initialization."""
+        try:
+            # Check if we have TypeScript/JavaScript files
+            ts_files = self._find_typescript_files()
+
+            if not ts_files:
+                logger.warning(f"No TypeScript/JavaScript files found in {self.project_path}")
+                return
+
+            logger.info(f"Found {len(ts_files)} TypeScript/JavaScript files")
+
+            # Notify workspace folders change
+            self._send_notification(
+                "workspace/didChangeWorkspaceFolders",
+                {
+                    "event": {
+                        "added": [{"uri": self.project_path.as_uri(), "name": self.project_path.name}],
+                        "removed": [],
+                    }
+                },
+            )
+
+            # Process configuration files
+            config_found = self._process_config_files()
+
+            # Bootstrap project by opening sample files
+            self._bootstrap_project(ts_files, config_found)
+
+        except Exception as e:
+            logger.warning(f"Failed to configure TypeScript workspace: {e}")
+
+    def _find_typescript_files(self) -> list:
+        """Find all TypeScript/JavaScript files in the project."""
+        all_files = []
+        for pattern in ["*.ts", "*.tsx", "*.js", "*.jsx"]:
+            all_files.extend(list(self.project_path.rglob(pattern)))
+
+        return self.filter_src_files(all_files)
+
+    def _process_config_files(self) -> bool:
+        """Process TypeScript configuration files and return True if any found."""
+        config_files = [
+            self.project_path / "tsconfig.json",
+            self.project_path / "jsconfig.json",
+            self.project_path / "package.json",
+        ]
+
+        config_found = False
+        for config_path in config_files:
+            if config_path.exists():
+                logger.info(f"Found configuration file: {config_path}")
+                config_found = True
+                self._send_notification(
+                    "workspace/didChangeWatchedFiles",
+                    {"changes": [{"uri": config_path.as_uri(), "type": 1}]},  # Created/Changed
+                )
+
+        return config_found
+
+    def _bootstrap_project(self, ts_files: list, config_found: bool):
+        """Bootstrap TypeScript project by opening files."""
+        logger.info("Opening sample files to bootstrap TypeScript project...")
+        # Files are already filtered in _find_typescript_files
+        sample_files = ts_files[:3]
+
+        # Open bootstrap files
+        for file_path in sample_files:
+            try:
+                content = file_path.read_text(encoding="utf-8")
+                file_uri = file_path.as_uri()
+                self._send_notification(
+                    "textDocument/didOpen",
+                    {"textDocument": {"uri": file_uri, "languageId": self.language_id, "version": 1, "text": content}},
+                )
+                logger.debug(f"Opened bootstrap file: {file_path}")
+            except Exception as e:
+                logger.debug(f"Could not open bootstrap file {file_path}: {e}")
+
+        # Wait for project initialization
+        wait_time = 5 if config_found else 8
+        logger.info(f"Waiting {wait_time}s for TypeScript server to initialize project...")
+        time.sleep(wait_time)
+
+        # Validate and close bootstrap files
+        if self._validate_typescript_project():
+            logger.info("TypeScript project successfully loaded!")
+        else:
+            logger.warning("TypeScript project still not loaded, but continuing...")
+
+        self._close_bootstrap_files(sample_files)
+
+    def _close_bootstrap_files(self, sample_files: list):
+        """Close bootstrap files that were opened for project initialization."""
+        for file_path in sample_files:
+            try:
+                self._send_notification("textDocument/didClose", {"textDocument": {"uri": file_path.as_uri()}})
+            except Exception:
+                pass
+
+    def _prepare_for_analysis(self):
+        """TypeScript-specific preparation before analysis."""
+        logger.info("Waiting additional time for TypeScript server to fully initialize...")
+        time.sleep(2)
+
+        if not self._validate_typescript_project():
+            logger.warning("TypeScript project not properly loaded. Analysis may be limited.")
+
+    def _validate_typescript_project(self) -> bool:
+        """Validate that TypeScript server has a project loaded."""
+        try:
+            logger.debug("Validating TypeScript project is loaded...")
+            params = {"query": "test"}
+            req_id = self._send_request("workspace/symbol", params)
+            response = self._wait_for_response(req_id)
+
+            if "error" in response:
+                error_msg = response["error"]
+                if "No Project" in str(error_msg):
+                    logger.error("TypeScript server reports 'No Project' - project not properly loaded")
+                    return False
+                else:
+                    logger.warning(f"workspace/symbol test failed but may work: {error_msg}")
+                    return True
+
+            logger.debug("TypeScript project validation successful")
+            return True
+
+        except Exception as e:
+            logger.error(f"Failed to validate TypeScript project: {e}")
+            return False

static_analyzer/programming_language.py

@@ -0,0 +1,152 @@
+import logging
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from static_analyzer.constants import Language
+
+logger = logging.getLogger(__name__)
+
+
+class LanguageConfig(BaseModel):
+    """Base configuration class for language-specific settings."""
+
+    model_config = {"frozen": True}  # Make configs immutable
+
+
+class JavaConfig(LanguageConfig):
+    """Java-specific configuration."""
+
+    jdtls_root: Path = Field(description="Path to the JDTLS (Java Language Server) installation directory")
+
+
+class ProgrammingLanguage:
+    def __init__(
+        self,
+        language: str,
+        size: int,
+        percentage: float,
+        suffixes: list[str],
+        server_commands: list[str] | None = None,
+        lsp_server_key: str | None = None,
+        language_specific_config: LanguageConfig | None = None,
+    ):
+        self.language = language
+        self.size = size
+        self.percentage = percentage
+        self.suffixes = suffixes
+        self.server_commands = server_commands
+        # group related languages (e.g., JS, TSX, JSX -> typescript) to the same language server
+        self.lsp_server_key = lsp_server_key or language.lower()
+        # Store language-specific configuration (e.g., JavaConfig for Java)
+        self.language_specific_config = language_specific_config
+
+    def get_suffix_pattern(self) -> list[str]:
+        """Generate and return pattern for the file suffixes, to use in .rglob(pattern)"""
+        if not self.suffixes:
+            return ["*"]
+        # Join suffixes with '|' to create a regex pattern
+        return [f"*.{suffix.lstrip('.')}" for suffix in self.suffixes]
+
+    def get_language_id(self) -> str:
+        # id for the language, used in LSP server
+        return self.language.lower().replace(" ", "_")
+
+    def get_server_parameters(self) -> list[str]:
+        if not self.server_commands:
+            raise ValueError(
+                f"No server commands defined for {self.language}. "
+                "Please ensure the language is supported and has server commands defined."
+            )
+        return self.server_commands
+
+    def is_supported_lang(self) -> bool:
+        return self.server_commands is not None
+
+    def __hash__(self):
+        return hash(self.lsp_server_key)
+
+    def __eq__(self, other):
+        if not isinstance(other, ProgrammingLanguage):
+            return False
+        return self.lsp_server_key == other.lsp_server_key
+
+    def __str__(self):
+        return f"ProgrammingLanguage(language={self.language}, lsp_server_key={self.lsp_server_key}, size={self.size}, percentage={self.percentage:.2f}%, suffixes={self.suffixes})"
+
+
+class ProgrammingLanguageBuilder:
+    """Builder to create ProgrammingLanguage instances from tokei output with greedy LSP matching."""
+
+    def __init__(self, lsp_configs: dict):
+        self.lsp_configs = lsp_configs
+        # Build reverse index: extension -> lsp_config_key
+        self._extension_to_lsp: dict[str, str] = {}
+        for lsp_server_key, config in lsp_configs.items():
+            for ext in config.get("file_extensions", []):
+                # Normalize extension (ensure it starts with '.')
+                normalized_ext = ext if ext.startswith(".") else f".{ext}"
+                self._extension_to_lsp[normalized_ext] = lsp_server_key
+
+    def _find_lsp_server_key(self, tokei_language: str, file_suffixes: set[str]) -> str | None:
+        """
+        Find the LSP config key for a tokei language by matching file extensions.
+
+        Args:
+            tokei_language: Language name from tokei output (e.g., "JavaScript", "TSX")
+            file_suffixes: Set of file suffixes from tokei reports
+
+        Returns:
+            LSP config key if found, None otherwise
+        """
+
+        # Try direct match with lsp_configs keys
+        normalized = tokei_language.lower()
+        if normalized in self.lsp_configs:
+            return normalized
+
+        # Fallback: try matching by file extensions
+        for suffix in file_suffixes:
+            normalized_suffix = suffix if suffix.startswith(".") else f".{suffix}"
+            if normalized_suffix in self._extension_to_lsp:
+                return self._extension_to_lsp[normalized_suffix]
+
+        return None
+
+    def build(
+        self,
+        tokei_language: str,
+        code_count: int,
+        percentage: float,
+        file_suffixes: set[str],
+    ) -> ProgrammingLanguage:
+        lsp_server_key = self._find_lsp_server_key(tokei_language, file_suffixes)
+
+        server_commands: list | None = None
+        config_suffixes: set[str] = set()
+        language_specific_config: LanguageConfig | None = None
+
+        if lsp_server_key and lsp_server_key in self.lsp_configs:
+            config = self.lsp_configs[lsp_server_key]
+            server_commands = config.get("command")
+            config_suffixes = set(config.get("file_extensions", []))
+
+            # Create language-specific config based on the LSP server key
+            if lsp_server_key == Language.JAVA and "jdtls_root" in config:
+                language_specific_config = JavaConfig(jdtls_root=Path(config["jdtls_root"]))
+
+        # Merge suffixes from tokei and config
+        all_suffixes = file_suffixes | config_suffixes
+
+        return ProgrammingLanguage(
+            language=tokei_language,
+            size=code_count,
+            percentage=percentage,
+            suffixes=list(all_suffixes),
+            server_commands=server_commands,
+            lsp_server_key=lsp_server_key,
+            language_specific_config=language_specific_config,
+        )
+
+    def get_supported_extensions(self) -> set[str]:
+        return set(self._extension_to_lsp.keys())

static_analyzer/reference_resolve_mixin.py

@@ -0,0 +1,166 @@
+import logging
+import logging
+import os
+from pathlib import Path
+from typing import Any
+
+from agents.agent_responses import AnalysisInsights
+from static_analyzer.analysis_result import StaticAnalysisResults
+
+logger = logging.getLogger(__name__)
+
+
+class ReferenceResolverMixin:
+    _parse_invoke: Any  # Provided by Agent base class
+
+    def __init__(self, repo_dir: Path, static_analysis: StaticAnalysisResults):
+        self.repo_dir = repo_dir
+        self.static_analysis = static_analysis
+
+    def fix_source_code_reference_lines(self, analysis: AnalysisInsights):
+        logger.info(f"Fixing source code reference lines for the analysis: {analysis.llm_str()}")
+        for component in analysis.components:
+            for reference in component.key_entities:
+                # Check if the file is already resolved
+                if reference.reference_file is not None and os.path.exists(reference.reference_file):
+                    continue
+
+                self._resolve_single_reference(reference, component.assigned_files)
+
+        # Remove unresolved references
+        self._remove_unresolved_references(analysis)
+
+        return self._relative_paths(analysis)
+
+    def _resolve_single_reference(self, reference, file_candidates: list[str] | None = None):
+        """Orchestrates different resolution strategies for a single reference."""
+        assert self.static_analysis is not None, "static_analysis required for reference resolution"
+        qname = reference.qualified_name.replace(os.sep, ".")
+
+        for lang in self.static_analysis.get_languages():
+            # Try exact match first
+            if self._try_exact_match(reference, qname, lang):
+                return
+
+            # Try loose matching
+            if self._try_loose_match(reference, qname, lang):
+                return
+
+            # Try file path resolution
+            if self._try_file_path_resolution(reference, qname, lang, file_candidates):
+                return
+
+        # No resolution found - will be cleaned up later
+        logger.warning(f"[Reference Resolution] Could not resolve reference {reference.qualified_name} in any language")
+
+    def _try_exact_match(self, reference, qname, lang):
+        """Attempts exact reference matching."""
+        try:
+            node = self.static_analysis.get_reference(lang, qname)
+            reference.reference_file = node.file_path
+            reference.reference_start_line = node.line_start + 1  # match 1 based indexing
+            reference.reference_end_line = node.line_end + 1  # match 1 based indexing
+            reference.qualified_name = qname
+            logger.info(
+                f"[Reference Resolution] Matched {reference.qualified_name} in {lang} at {reference.reference_file}"
+            )
+            return True
+        except (ValueError, FileExistsError) as e:
+            logger.warning(f"[Reference Resolution] Exact match failed for {reference.qualified_name} in {lang}: {e}")
+            return False
+
+    def _try_loose_match(self, reference, qname, lang):
+        """Attempts loose reference matching."""
+        try:
+            _, node = self.static_analysis.get_loose_reference(lang, qname)
+            if node is not None:
+                reference.reference_file = node.file_path
+                reference.reference_start_line = node.line_start + 1
+                reference.reference_end_line = node.line_end + 1
+                reference.qualified_name = qname
+                logger.info(
+                    f"[Reference Resolution] Loosely matched {reference.qualified_name} in {lang} at {reference.reference_file}"
+                )
+                return True
+        except Exception as e:
+            logger.warning(f"[Reference Resolution] Loose match failed for {qname} in {lang}: {e}")
+        return False
+
+    def _try_file_path_resolution(self, reference, qname, lang, file_candidates: list[str] | None = None):
+        """Attempts to resolve reference through file path matching."""
+        # First try existing reference file path
+        if self._try_existing_reference_file(reference, lang):
+            return True
+
+        # Then try qualified name as file path
+        return self._try_qualified_name_as_path(reference, qname, lang, file_candidates)
+
+    def _try_existing_reference_file(self, reference, lang):
+        """Tries to resolve using existing reference file path."""
+        if (reference.reference_file is not None) and (not Path(reference.reference_file).is_absolute()):
+            joined_path = os.path.join(self.repo_dir, reference.reference_file)
+            if os.path.exists(joined_path):
+                reference.reference_file = joined_path
+                logger.info(
+                    f"[Reference Resolution] File path matched for {reference.qualified_name} in {lang} at {reference.reference_file}"
+                )
+                return True
+            else:
+                reference.reference_file = None
+        return False
+
+    def _try_qualified_name_as_path(self, reference, qname, lang, file_candidates: list[str] | None = None):
+        """Tries to resolve qualified name as various file path patterns."""
+        file_path = qname.replace(".", os.sep)  # Get file path
+        full_path = os.path.join(self.repo_dir, file_path)
+        file_ref = ".".join(full_path.rsplit(os.sep, 1))
+        extra_paths = file_candidates or []
+        paths = [full_path, f"{file_path}.py", f"{file_path}.ts", f"{file_path}.tsx", file_ref, *extra_paths]
+
+        for path in paths:
+            if os.path.exists(path):
+                reference.reference_file = str(path)
+                logger.info(
+                    f"[Reference Resolution] Path matched for {reference.qualified_name} in {lang} at {reference.reference_file}"
+                )
+                return True
+        return False
+
+    def _remove_unresolved_references(self, analysis: AnalysisInsights):
+        """Remove references and assigned files that couldn't be resolved to existing files."""
+        for component in analysis.components:
+            # Remove unresolved key_entities
+            original_ref_count = len(component.key_entities)
+            component.key_entities = [
+                ref
+                for ref in component.key_entities
+                if ref.reference_file is not None and os.path.exists(ref.reference_file)
+            ]
+            removed_ref_count = original_ref_count - len(component.key_entities)
+            if removed_ref_count > 0:
+                logger.info(
+                    f"[Reference Resolution] Removed {removed_ref_count} unresolved reference(s) "
+                    f"from component '{component.name}'"
+                )
+
+            # Remove unresolved assigned_files
+            original_file_count = len(component.assigned_files)
+            component.assigned_files = [
+                f
+                for f in component.assigned_files
+                if os.path.exists(os.path.join(self.repo_dir, f)) or os.path.exists(f)
+            ]
+            removed_file_count = original_file_count - len(component.assigned_files)
+            if removed_file_count > 0:
+                logger.info(
+                    f"[Reference Resolution] Removed {removed_file_count} unresolved assigned file(s) "
+                    f"from component '{component.name}'"
+                )
+
+    def _relative_paths(self, analysis: AnalysisInsights):
+        """Convert all reference file paths to relative paths."""
+        for component in analysis.components:
+            for reference in component.key_entities:
+                if reference.reference_file and reference.reference_file.startswith(str(self.repo_dir)):
+                    reference.reference_file = os.path.relpath(reference.reference_file, self.repo_dir)
+        return analysis

static_analyzer/scanner.py

@@ -0,0 +1,95 @@
+import json
+import logging
+import subprocess
+from pathlib import Path
+from typing import Set
+
+from static_analyzer.programming_language import ProgrammingLanguage, ProgrammingLanguageBuilder
+from utils import get_config
+
+logger = logging.getLogger(__name__)
+
+
+class ProjectScanner:
+    def __init__(self, repo_location: Path):
+        self.repo_location = repo_location
+        self.all_text_files: list[str] = []
+
+    def scan(self) -> list[ProgrammingLanguage]:
+        """
+        Scan the repository using Tokei and return parsed results.
+
+        Also populates self.all_text_files with all text file paths found by Tokei.
+
+        Returns:
+            list[ProgrammingLanguage]: technologies with their sizes, percentages, and suffixes
+        """
+
+        commands = get_config("tools")["tokei"]["command"]
+        result = subprocess.run(commands, cwd=self.repo_location, capture_output=True, text=True, check=True)
+
+        server_config = get_config("lsp_servers")
+        builder = ProgrammingLanguageBuilder(server_config)
+
+        # Parse Tokei JSON output
+        tokei_data = json.loads(result.stdout)
+
+        # Compute total code count
+        total_code = tokei_data.get("Total", {}).get("code", 0)
+        if not total_code:
+            logger.warning("No total code count found in Tokei output")
+            return []
+
+        programming_languages: list[ProgrammingLanguage] = []
+        all_files: list[str] = []
+        for technology, stats in tokei_data.items():
+            if technology == "Total":
+                continue
+
+            # Collect ALL text file paths from Tokei for file coverage,
+            # including languages with code_count == 0 (e.g. Markdown is 100% comments)
+            for report in stats.get("reports", []):
+                all_files.append(report["name"])
+
+            code_count = stats.get("code", 0)
+            if code_count == 0:
+                continue
+
+            percentage = code_count / total_code * 100
+
+            # Extract suffixes from reports
+            suffixes = set()
+            for report in stats.get("reports", []):
+                suffixes |= self._extract_suffixes([report["name"]])
+
+            pl = builder.build(
+                tokei_language=technology,
+                code_count=code_count,
+                percentage=percentage,
+                file_suffixes=suffixes,
+            )
+
+            logger.debug(f"Found: {pl}")
+            if pl.percentage >= 1:
+                programming_languages.append(pl)
+
+        self.all_text_files = all_files
+        return programming_languages
+
+    @staticmethod
+    def _extract_suffixes(files: list[str]) -> Set[str]:
+        """
+        Extract unique file suffixes from a list of files.
+
+        Args:
+            files (list[str]): list of file paths
+
+        Returns:
+            Set[str]: Unique file extensions/suffixes
+        """
+        suffixes = set()
+        for file_path in files:
+            suffix = Path(file_path).suffix
+            if suffix:  # Only add non-empty suffixes
+                suffixes.add(suffix)
+        return suffixes