codeboarding 0.9.0__py3-none-any.whl

agents/agent_responses.py

@@ -0,0 +1,363 @@
+import abc
+import hashlib
+import logging
+from abc import abstractmethod
+from typing import get_origin, Optional
+
+from pydantic import BaseModel, Field
+
+logger = logging.getLogger(__name__)
+
+ROOT_PARENT_ID = "ROOT"
+COMPONENT_ID_BYTES = 8
+
+
+class LLMBaseModel(BaseModel, abc.ABC):
+    """Base model for LLM-parseable response types."""
+
+    @abstractmethod
+    def llm_str(self):
+        raise NotImplementedError("LLM String has to be implemented.")
+
+    @classmethod
+    def extractor_str(cls):
+        # Here iterate over the fields that we have and use their description like:
+        result_str = "please extract the following: "
+        for fname, fvalue in cls.model_fields.items():
+            # check if the field type is Optional
+            ftype = fvalue.annotation
+            # Check if the type is a typing.List (e.g., typing.List[SomeType])
+            if get_origin(ftype) is list:
+                # get the type of the list:
+                if ftype is not None and hasattr(ftype, "__args__"):
+                    ftype = ftype.__args__[0]
+                result_str += f"{fname} which is a list ("
+            if ftype is Optional:
+                result_str += f"{fname} ({fvalue.description}), "
+            elif ftype is not None and isinstance(ftype, type) and issubclass(ftype, LLMBaseModel):
+                # Now I need to call the extractor_str method of the field
+                result_str += ftype.extractor_str()
+            else:
+                result_str += f"{fname} ({fvalue.description}), "
+            if get_origin(ftype) is list:
+                result_str += "), "
+        return result_str
+
+
+class SourceCodeReference(LLMBaseModel):
+    """Reference to source code including qualified name and file location."""
+
+    qualified_name: str = Field(
+        description="Qualified name of the source code, e.g., `langchain.tools.tool` or `langchain_core.output_parsers.JsonOutputParser` or `langchain_core.output_parsers.JsonOutputParser:parse`."
+    )
+
+    reference_file: str | None = Field(
+        default=None,
+        description="File path where the source code is located, e.g., `langchain/tools/tool.py` or `langchain_core/output_parsers/json_output_parser.py`.",
+    )
+
+    reference_start_line: int | None = Field(
+        default=None,
+        description="The line number in the source code where the reference starts. Only if you are absolutely sure add this, otherwise None.",
+    )
+    reference_end_line: int | None = Field(
+        default=None,
+        description="The line number in the source code where the reference ends. Only if you are absolutely sure add this, otherwise None.",
+    )
+
+    def llm_str(self):
+        if self.reference_start_line is None or self.reference_end_line is None:
+            return f"QName:`{self.qualified_name}` FileRef: `{self.reference_file}`"
+        if (
+            self.reference_start_line <= self.reference_end_line <= 0
+            or self.reference_start_line == self.reference_end_line
+        ):
+            return f"QName:`{self.qualified_name}` FileRef: `{self.reference_file}`"
+        return f"QName:`{self.qualified_name}` FileRef: `{self.reference_file}`, Lines:({self.reference_start_line}:{self.reference_end_line})"
+
+    def __str__(self):
+        if self.reference_start_line is None or self.reference_end_line is None:
+            return f"`{self.qualified_name}`"
+        if (
+            self.reference_start_line <= self.reference_end_line <= 0
+            or self.reference_start_line == self.reference_end_line
+        ):
+            return f"`{self.qualified_name}`"
+        return f"`{self.qualified_name}`:{self.reference_start_line}-{self.reference_end_line}"
+
+
+class Relation(LLMBaseModel):
+    """A relationship between two components."""
+
+    relation: str = Field(description="Single phrase used for the relationship of two components.")
+    src_name: str = Field(description="Source component name")
+    dst_name: str = Field(description="Target component name")
+    src_id: str = Field(default="", description="Component ID of the source.", exclude=True)
+    dst_id: str = Field(default="", description="Component ID of the destination.", exclude=True)
+
+    def llm_str(self):
+        return f"({self.src_name}, {self.relation}, {self.dst_name})"
+
+
+class ClustersComponent(LLMBaseModel):
+    """A grouped component from cluster analysis - may contain multiple clusters."""
+
+    cluster_ids: list[int] = Field(
+        description="List of cluster IDs from the CFG analysis that are grouped together (e.g., [1, 3, 5])"
+    )
+    description: str = Field(
+        description="Explanation of what this component does, its main flow, and WHY these clusters are grouped together"
+    )
+
+    def llm_str(self):
+        ids_str = ", ".join(str(cid) for cid in self.cluster_ids)
+        return f"**Clusters [{ids_str}]**\n   {self.description}"
+
+
+class ClusterAnalysis(LLMBaseModel):
+    """Analysis results containing grouped cluster components."""
+
+    cluster_components: list[ClustersComponent] = Field(
+        description="Grouped clusters into logical components. Multiple cluster IDs can be grouped together if they work as a cohesive unit."
+    )
+
+    def llm_str(self):
+        if not self.cluster_components:
+            return "No clusters analyzed."
+        title = "# Grouped Cluster Components\n"
+        body = "\n".join(cc.llm_str() for cc in self.cluster_components)
+        return title + body
+
+
+class Component(LLMBaseModel):
+    """A software component with name, description, and key entities."""
+
+    name: str = Field(description="Name of the component")
+    description: str = Field(description="A short description of the component.")
+
+    # LLM picks these: The MOST IMPORTANT/critical methods and classes
+    key_entities: list[SourceCodeReference] = Field(
+        description="The most important/critical classes and methods that represent this component's core functionality. Pick 2-5 key entities."
+    )
+
+    # Deterministic from static analysis: ALL files belonging to this component
+    assigned_files: list[str] = Field(
+        description="All source files assigned to this component (populated deterministically).",
+        default_factory=list,
+        exclude=True,
+    )
+
+    source_cluster_ids: list[int] = Field(
+        description="List of cluster IDs from CFG analysis that this component encompasses.",
+        default_factory=list,
+    )
+
+    component_id: str = Field(
+        default="",
+        description="Deterministic unique identifier for this component.",
+        exclude=True,
+    )
+
+    def llm_str(self):
+        n = f"**Component:** `{self.name}`"
+        d = f"   - *Description*: {self.description}"
+        qn = ""
+        if self.key_entities:
+            qn += "   - *Key Entities*: "
+            qn += ", ".join(f"`{q.llm_str()}`" for q in self.key_entities)
+        return "\n".join([n, d, qn]).strip()
+
+
+class AnalysisInsights(LLMBaseModel):
+    """Project analysis insights including components and their relations."""
+
+    description: str = Field(
+        description="One paragraph explaining the functionality which is represented by this graph. What the main flow is and what is its purpose."
+    )
+    components: list[Component] = Field(description="List of the components identified in the project.")
+    components_relations: list[Relation] = Field(description="List of relations among the components.")
+
+    def llm_str(self):
+        if not self.components:
+            return "No abstract components found."
+        title = "# 📦 Abstract Components Overview\n"
+        body = "\n".join(ac.llm_str() for ac in self.components)
+        relations = "\n".join(cr.llm_str() for cr in self.components_relations)
+        return title + body + relations
+
+
+def hash_component_id(parent_id: str, name: str, sibling_index: int = 0) -> str:
+    """Hash a deterministic component ID from parent ID, name, and sibling index.
+
+    Note:
+        The ID is a compact, 64-bit prefix of SHA-256 (8 bytes -> 16 hex chars).
+        Truncation happens at the byte level to keep the representation explicit.
+    """
+    raw = f"{parent_id}:{name}:{sibling_index}".encode("utf-8")
+    return hashlib.sha256(raw).digest()[:COMPONENT_ID_BYTES].hex()
+
+
+def assign_component_ids(analysis: AnalysisInsights, parent_id: str = ROOT_PARENT_ID) -> None:
+    """Assign deterministic component IDs to all components in an analysis.
+
+    Handles same-named siblings by using a sibling index tiebreaker.
+    """
+    name_counts: dict[str, int] = {}
+    for component in analysis.components:
+        count = name_counts.get(component.name, 0)
+        component.component_id = hash_component_id(parent_id, component.name, count)
+        name_counts[component.name] = count + 1
+
+    # Assign relation IDs by looking up component names (first occurrence wins for duplicates)
+    name_to_id: dict[str, str] = {}
+    for c in analysis.components:
+        if c.name in name_to_id:
+            logger.warning(
+                f"Duplicate component name '{c.name}' found during ID assignment; "
+                f"relation lookup will use the first occurrence (ID: {name_to_id[c.name]})"
+            )
+        else:
+            name_to_id[c.name] = c.component_id
+    for relation in analysis.components_relations:
+        relation.src_id = name_to_id.get(relation.src_name, "")
+        relation.dst_id = name_to_id.get(relation.dst_name, "")
+
+
+class CFGComponent(LLMBaseModel):
+    """A component derived from control flow graph analysis."""
+
+    name: str = Field(description="Name of the abstract component")
+    description: str = Field(description="One paragraph explaining the component.")
+    referenced_source: list[str] = Field(
+        description="List of the qualified names of the methods and classes that are within this component."
+    )
+
+    def llm_str(self):
+        n = f"**Component:** `{self.name}`"
+        d = f"   - *Description*: {self.description}"
+        qn = ""
+        if self.referenced_source:
+            qn += "   - *Related Classes/Methods*: "
+            qn += ", ".join(f"`{q}`" for q in self.referenced_source)
+        return "\n".join([n, d, qn]).strip()
+
+
+class CFGAnalysisInsights(LLMBaseModel):
+    """Insights from control flow graph analysis including components and relations."""
+
+    components: list[CFGComponent] = Field(description="List of components identified in the CFG.")
+    components_relations: list[Relation] = Field(description="List of relations among the components in the CFG.")
+
+    def llm_str(self):
+        if not self.components:
+            return "No abstract components found in the CFG."
+        title = "# 📦 Abstract Components Overview from CFG\n"
+        body = "\n".join(ac.llm_str() for ac in self.components)
+        relations = "\n".join(cr.llm_str() for cr in self.components_relations)
+        return title + body + relations
+
+
+class ExpandComponent(LLMBaseModel):
+    """Decision on whether to expand a component with reasoning."""
+
+    should_expand: bool = Field(description="Whether the component should be expanded in detail or not.")
+    reason: str = Field(description="Reasoning behind the decision to expand or not.")
+
+    def llm_str(self):
+        return f"- *Should Expand:* {self.should_expand}\n- *Reason:* {self.reason}"
+
+
+class ValidationInsights(LLMBaseModel):
+    """Validation results with status and additional information."""
+
+    is_valid: bool = Field(description="Indicates whether the validation results in valid or not.")
+    additional_info: str | None = Field(
+        default=None,
+        description="Any additional information or context related to the validation.",
+    )
+
+    def llm_str(self):
+        return f"**Feedback Information:**\n{self.additional_info}"
+
+
+class UpdateAnalysis(LLMBaseModel):
+    """Feedback on how much a diagram needs updating."""
+
+    update_degree: int = Field(
+        description="Degree to which the diagram needs update. 0 means no update, 10 means complete update."
+    )
+    feedback: str = Field(description="Feedback provided on the analysis.")
+
+    def llm_str(self):
+        return f"**Feedback:**\n{self.feedback}"
+
+
+class MetaAnalysisInsights(LLMBaseModel):
+    """Insights from analyzing project metadata including type, domain, and architecture."""
+
+    project_type: str = Field(
+        description="Type/category of the project (e.g., web framework, data processing, ML library, etc.)"
+    )
+    domain: str = Field(
+        description="Domain or field the project belongs to (e.g., web development, data science, DevOps, etc.)"
+    )
+    architectural_patterns: list[str] = Field(description="Main architectural patterns typically used in such projects")
+    expected_components: list[str] = Field(description="Expected high-level components/modules based on project type")
+    technology_stack: list[str] = Field(description="Main technologies, frameworks, and libraries used")
+    architectural_bias: str = Field(
+        description="Guidance on how to interpret and organize components for this project type"
+    )
+
+    def llm_str(self):
+        title = "# 🎯 Project Metadata Analysis\n"
+        content = f"""
+**Project Type:** {self.project_type}
+**Domain:** {self.domain}
+**Technology Stack:** {", ".join(self.technology_stack)}
+**Architectural Patterns:** {", ".join(self.architectural_patterns)}
+**Expected Components:** {", ".join(self.expected_components)}
+**Architectural Bias:** {self.architectural_bias}
+"""
+        return title + content
+
+
+class FileClassification(LLMBaseModel):
+    """Classification of a file to a component."""
+
+    component_name: str = Field(description="Name of the component or module")
+    file_path: str = Field(description="Path to the file")
+
+    def llm_str(self):
+        return f"`{self.file_path}` -> Component: `{self.component_name}`"
+
+
+class ComponentFiles(LLMBaseModel):
+    """Collection of file classifications for components."""
+
+    file_paths: list[FileClassification] = Field(
+        description="All files with their classifications for each of the files assigned to a component."
+    )
+
+    def llm_str(self):
+        if not self.file_paths:
+            return "No files classified."
+        title = "# 📄 Component File Classifications\n"
+        body = "\n".join(f"- `{fc.file_path}` -> Component: `{fc.component_name}`" for fc in self.file_paths)
+        return title + body
+
+
+class FilePath(LLMBaseModel):
+    """File path with optional line range reference."""
+
+    file_path: str = Field(description="Full file path for the reference")
+    start_line: int | None = Field(
+        default=None,
+        description="Starting line number in the file for the reference (if applicable).",
+    )
+    end_line: int | None = Field(
+        default=None,
+        description="Ending line number in the file for the reference (if applicable).",
+    )
+
+    def llm_str(self):
+        return f"`{self.file_path}`: ({self.start_line}:{self.end_line})"

agents/cluster_methods_mixin.py

@@ -0,0 +1,281 @@
+import logging
+import os
+from pathlib import Path
+
+from agents.agent_responses import Component, AnalysisInsights
+from static_analyzer.analysis_result import StaticAnalysisResults
+from static_analyzer.graph import ClusterResult
+from static_analyzer.cluster_helpers import get_files_for_cluster_ids, get_all_cluster_ids
+
+logger = logging.getLogger(__name__)
+
+
+class ClusterMethodsMixin:
+    """
+    Mixin providing shared cluster-related functionality for agents.
+
+    This mixin provides methods for:
+    - Building cluster strings from CFG analysis (using CallGraph.cluster())
+    - Assigning files to components based on clusters and key_entities
+    - Ensuring unique key entities across components
+
+    All clustering logic is delegated to CallGraph.cluster() which provides:
+    - Deterministic cluster IDs (seed=42)
+    - Cached results
+    - File <-> cluster bidirectional mappings
+
+    IMPORTANT: All methods are stateless with respect to ClusterResult.
+    Cluster results must be passed explicitly as parameters.
+    """
+
+    # These attributes must be provided by the class using this mixin
+    repo_dir: Path
+    static_analysis: StaticAnalysisResults
+
+    def _get_files_for_clusters(self, cluster_ids: list[int], cluster_results: dict[str, ClusterResult]) -> set[str]:
+        """
+        Get all files that belong to the given cluster IDs.
+
+        Args:
+            cluster_ids: List of cluster IDs to get files for
+            cluster_results: dict mapping language -> ClusterResult
+
+        Returns:
+            Set of file paths
+        """
+        return get_files_for_cluster_ids(cluster_ids, cluster_results)
+
+    def _build_cluster_string(
+        self,
+        programming_langs: list[str],
+        cluster_results: dict[str, ClusterResult],
+        cluster_ids: set[int] | None = None,
+    ) -> str:
+        """
+        Build a cluster string for LLM consumption using pre-computed cluster results.
+
+        Args:
+            programming_langs: List of languages to include
+            cluster_results: Pre-computed cluster results mapping language -> ClusterResult
+            cluster_ids: Optional set of cluster IDs to filter by
+
+        Returns:
+            Formatted cluster string with headers per language
+        """
+        cluster_lines = []
+
+        for lang in programming_langs:
+            cfg = self.static_analysis.get_cfg(lang)
+            # Get cluster result for this language
+            cluster_result = cluster_results.get(lang)
+            cluster_str = cfg.to_cluster_string(cluster_ids, cluster_result)
+
+            if cluster_str.strip() and cluster_str not in ("empty", "none", "No clusters found."):
+                header = "Component CFG" if cluster_ids else "Clusters"
+                cluster_lines.append(f"\n## {lang.capitalize()} - {header}\n")
+                cluster_lines.append(cluster_str)
+                cluster_lines.append("\n")
+
+        return "".join(cluster_lines)
+
+    def _assign_files_to_component(self, component: Component, cluster_results: dict[str, ClusterResult]) -> None:
+        """
+        Assign files to a component.
+        1. Get all files from component's clusters (instant lookup)
+        2. Add resolved key_entity files
+        3. Convert to relative paths
+
+        Args:
+            component: Component to assign files to
+            cluster_results: dict mapping language -> ClusterResult
+        """
+        assigned: set[str] = set()
+
+        # Step 1: Files from clusters
+        if component.source_cluster_ids:
+            cluster_files = self._get_files_for_clusters(component.source_cluster_ids, cluster_results)
+            assigned.update(cluster_files)
+
+        # Step 2: Files from key_entities (already resolved by ReferenceResolverMixin)
+        for entity in component.key_entities:
+            if entity.reference_file:
+                # Handle both absolute and relative paths
+                if os.path.isabs(entity.reference_file):
+                    assigned.add(entity.reference_file)
+                else:
+                    abs_path = os.path.join(self.repo_dir, entity.reference_file)
+                    if os.path.exists(abs_path):
+                        assigned.add(abs_path)
+                    else:
+                        assigned.add(entity.reference_file)
+
+        # Convert to relative paths
+        component.assigned_files = [os.path.relpath(f, self.repo_dir) if os.path.isabs(f) else f for f in assigned]
+
+    def _ensure_unique_key_entities(self, analysis: AnalysisInsights):
+        """
+        Ensure that key_entities are unique across components.
+
+        If a key_entity (identified by qualified_name) appears in multiple components,
+        keep it only in the component where it's most relevant:
+        1. If it's in the component's assigned_files -> keep it there (highest priority)
+        2. Otherwise, keep it in the first component that references it
+
+        This prevents confusion in documentation where the same class/method
+        is listed as a "key entity" for multiple components.
+        """
+        logger.info("[ClusterMethodsMixin] Ensuring key_entities are unique across components")
+
+        seen_entities: dict[str, Component] = {}
+
+        for component in analysis.components:
+            if component.name == "Unclassified":
+                continue
+
+            entities_to_remove = []
+
+            for key_entity in component.key_entities:
+                qname = key_entity.qualified_name
+
+                if qname in seen_entities:
+                    original_component = seen_entities[qname]
+                    ref_file = key_entity.reference_file
+
+                    current_has_file = ref_file and any(
+                        ref_file in assigned_file for assigned_file in component.assigned_files
+                    )
+                    original_has_file = ref_file and any(
+                        ref_file in assigned_file for assigned_file in original_component.assigned_files
+                    )
+
+                    if current_has_file and not original_has_file:
+                        # Move to current component
+                        original_component.key_entities = [
+                            e for e in original_component.key_entities if e.qualified_name != qname
+                        ]
+                        seen_entities[qname] = component
+                        logger.debug(
+                            f"[ClusterMethodsMixin] Moved key_entity '{qname}' from {original_component.name} to {component.name}"
+                        )
+                    else:
+                        # Keep in original component
+                        entities_to_remove.append(key_entity)
+                        logger.debug(
+                            f"[ClusterMethodsMixin] Removed duplicate key_entity '{qname}' from {component.name} (kept in {original_component.name})"
+                        )
+                else:
+                    seen_entities[qname] = component
+
+            component.key_entities = [e for e in component.key_entities if e not in entities_to_remove]
+
+    def _ensure_unique_file_assignments(self, analysis: AnalysisInsights) -> None:
+        """
+        Deduplicate assigned_files within each component.
+
+        A file may legitimately appear in multiple components, but should not
+        appear more than once within the same component's assigned_files list.
+        """
+        logger.info("[ClusterMethodsMixin] Deduplicating file assignments within components")
+
+        total_removed = 0
+
+        for component in analysis.components:
+            seen: set[str] = set()
+            unique_files: list[str] = []
+            for file_path in component.assigned_files:
+                if file_path in seen:
+                    logger.debug(
+                        f"[ClusterMethodsMixin] Removed duplicate file '{file_path}' within '{component.name}'"
+                    )
+                    total_removed += 1
+                else:
+                    seen.add(file_path)
+                    unique_files.append(file_path)
+
+            component.assigned_files = unique_files
+
+        if total_removed > 0:
+            logger.info(f"[ClusterMethodsMixin] Removed {total_removed} duplicate file assignment(s)")
+
+    def _sanitize_component_cluster_ids(
+        self,
+        analysis: AnalysisInsights,
+        valid_cluster_ids: set[int] | None = None,
+        cluster_results: dict[str, ClusterResult] | None = None,
+    ) -> None:
+        """
+        Sanitize cluster IDs in the analysis by removing invalid ones.
+        Removes cluster IDs that don't exist in the static analysis.
+
+        Args:
+            analysis: The analysis to sanitize
+            valid_cluster_ids: Optional set of valid IDs. If None, derives from cluster_results.
+            cluster_results: dict mapping language -> ClusterResult. Required if valid_cluster_ids is None.
+        """
+        if valid_cluster_ids is None:
+            if cluster_results is None:
+                logger.error("Must provide either valid_cluster_ids or cluster_results")
+                return
+            valid_cluster_ids = get_all_cluster_ids(cluster_results)
+
+        for component in analysis.components:
+            if component.source_cluster_ids:
+                original_ids = component.source_cluster_ids.copy()
+                component.source_cluster_ids = [cid for cid in component.source_cluster_ids if cid in valid_cluster_ids]
+                removed_ids = set(original_ids) - set(component.source_cluster_ids)
+                if removed_ids:
+                    logger.warning(
+                        f"[ClusterMethodsMixin] Removed invalid cluster IDs {removed_ids} from component '{component.name}'"
+                    )
+
+    def _create_strict_component_subgraph(self, component: Component) -> tuple[str, dict]:
+        """
+        Create a strict subgraph containing ONLY nodes from the component's assigned files.
+        This ensures the analysis is strictly scoped to the component's boundaries.
+
+        Args:
+            component: Component with assigned_files to filter by
+
+        Returns:
+            Tuple of (formatted cluster string, cluster_results dict)
+            where cluster_results maps language -> ClusterResult for the subgraph
+        """
+        if not component.assigned_files:
+            logger.warning(f"[ClusterMethodsMixin] Component {component.name} has no assigned_files")
+            return "No assigned files found for this component.", {}
+
+        # Convert assigned files to absolute paths for comparison
+        assigned_file_set = set()
+        for f in component.assigned_files:
+            abs_path = os.path.join(self.repo_dir, f) if not os.path.isabs(f) else f
+            assigned_file_set.add(abs_path)
+
+        result_parts = []
+        cluster_results = {}
+
+        for lang in self.static_analysis.get_languages():
+            cfg = self.static_analysis.get_cfg(lang)
+
+            # Use strict filtering logic
+            sub_cfg = cfg.filter_by_files(assigned_file_set)
+
+            if sub_cfg.nodes:
+                # Calculate clusters for the subgraph
+                sub_cluster_result = sub_cfg.cluster()
+                cluster_results[lang] = sub_cluster_result
+
+                cluster_str = sub_cfg.to_cluster_string(cluster_result=sub_cluster_result)
+                if cluster_str.strip() and cluster_str not in ("empty", "none", "No clusters found."):
+                    result_parts.append(f"\n## {lang.capitalize()} - Component CFG\n")
+                    result_parts.append(cluster_str)
+                    result_parts.append("\n")
+
+        result = "".join(result_parts)
+
+        if not result.strip():
+            logger.warning(
+                f"[ClusterMethodsMixin] No CFG found for component {component.name} with {len(component.assigned_files)} assigned files"
+            )
+            return "No relevant CFG clusters found for this component.", cluster_results
+
+        return result, cluster_results

agents/constants.py

@@ -0,0 +1,13 @@
+"""Constants for the agents module."""
+
+
+class LLMDefaults:
+    DEFAULT_AGENT_TEMPERATURE = 0.1
+    DEFAULT_PARSING_TEMPERATURE = 0
+    AWS_MAX_TOKENS = 4096
+
+
+class FileStructureConfig:
+    MAX_LINES = 500
+    DEFAULT_MAX_DEPTH = 10
+    FALLBACK_MAX_LINES = 50000