codeboarding 0.12.4__tar.gz → 0.13.0__tar.gz

{codeboarding-0.12.4/codeboarding.egg-info → codeboarding-0.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codeboarding
-Version: 0.12.4
+Version: 0.13.0
 Summary: Interactive Diagrams for Code
 Author: CodeBoarding Team
 License-Expression: MIT

{codeboarding-0.12.4 → codeboarding-0.13.0}/README.md

@@ -137,6 +137,14 @@ python main.py partial --local ./my-project --component-id "1.2"
 python main.py full https://github.com/pytorch/pytorch
 ```
 
+> **Incremental needs a baseline.** `incremental` diffs the working tree against the previous
+> analysis in `.codeboarding/` (`analysis.json` + `fingerprint.json`). That baseline can live
+> purely locally — a prior `full`/`incremental` run in the same output dir is enough. Commit
+> `.codeboarding/` only if you want the baseline to travel with the branch (so a teammate or a
+> fresh checkout can run incremental too). With no baseline at all — or one that predates content
+> versioning — `incremental` fails fast with "run a full analysis first" rather than silently
+> doing a full run.
+
 ## Where to use it
 
 - [CLI](https://github.com/CodeBoarding/CodeBoarding) for local analysis, automation, and CI workflows.

{codeboarding-0.12.4 → codeboarding-0.13.0}/agents/abstraction_agent.py

@@ -7,22 +7,31 @@ from langchain_core.prompts import PromptTemplate
 from agents.agent import CodeBoardingAgent
 from agents.agent_responses import (
     AnalysisInsights,
+    ComponentApiSurfaces,
+    ComponentArchitecture,
+    ComponentRelations,
     ClusterAnalysis,
     MetaAnalysisInsights,
     assign_component_ids,
+    assign_relation_ids,
 )
 from agents.cluster_methods_mixin import ClusterMethodsMixin
 from agents.prompts import (
     get_cluster_grouping_message,
     get_final_analysis_message,
+    get_api_surfaces_message,
+    get_relation_analysis_message,
     get_system_message,
+    format_project_system_message,
 )
+from agents.relation_edges import index_relation_endpoints
+from agents.repair import ComponentRepairContext, repair_component_group_names, repair_key_entities
 from agents.validation import (
     ValidationContext,
     validate_cluster_coverage,
     validate_group_name_coverage,
     validate_key_entities,
-    validate_relation_component_names,
+    validate_relations,
 )
 from monitoring import trace
 from static_analyzer.analysis_result import StaticAnalysisResults
@@ -45,7 +54,8 @@ class AbstractionAgent(ClusterMethodsMixin, CodeBoardingAgent):
         agent_llm: BaseChatModel,
         parsing_llm: BaseChatModel,
     ):
-        super().__init__(repo_dir, static_analysis, get_system_message(), agent_llm, parsing_llm)
+        system_message = format_project_system_message(get_system_message(), project_name, meta_context)
+        super().__init__(repo_dir, static_analysis, system_message, agent_llm, parsing_llm)
 
         self.project_name = project_name
         self.meta_context = meta_context
@@ -53,11 +63,26 @@ class AbstractionAgent(ClusterMethodsMixin, CodeBoardingAgent):
         self.prompts = {
             "group_clusters": PromptTemplate(
                 template=get_cluster_grouping_message(),
-                input_variables=["project_name", "cfg_clusters", "meta_context", "project_type"],
+                input_variables=["cfg_clusters"],
             ),
             "final_analysis": PromptTemplate(
                 template=get_final_analysis_message(),
-                input_variables=["project_name", "cluster_analysis", "meta_context", "project_type"],
+                input_variables=["cluster_analysis"],
+            ),
+            "api_surfaces": PromptTemplate(
+                template=get_api_surfaces_message(),
+                input_variables=[
+                    "component_summaries",
+                    "static_call_evidence",
+                ],
+            ),
+            "relation_analysis": PromptTemplate(
+                template=get_relation_analysis_message(),
+                input_variables=[
+                    "component_summaries",
+                    "api_surfaces",
+                    "static_call_evidence",
+                ],
             ),
         }
 
@@ -65,9 +90,6 @@ class AbstractionAgent(ClusterMethodsMixin, CodeBoardingAgent):
     def step_clusters_grouping(self, cluster_results: dict[str, ClusterResult]) -> ClusterAnalysis:
         logger.info(f"[AbstractionAgent] Grouping CFG clusters for: {self.project_name}")
 
-        meta_context_str = self.meta_context.llm_str() if self.meta_context else "No project context available."
-        project_type = self.meta_context.project_type if self.meta_context else "unknown"
-
         programming_langs = self.static_analysis.get_languages()
 
         # Measure everything that wraps cfg_clusters (system message + rendered
@@ -75,10 +97,7 @@ class AbstractionAgent(ClusterMethodsMixin, CodeBoardingAgent):
         # the input window before budgeting the cluster string.
         overhead_chars = len(str(self.system_message.content)) + len(
             self.prompts["group_clusters"].format(
-                project_name=self.project_name,
                 cfg_clusters="",
-                meta_context=meta_context_str,
-                project_type=project_type,
             )
         )
         cluster_str = self._build_cluster_string(
@@ -86,17 +105,14 @@ class AbstractionAgent(ClusterMethodsMixin, CodeBoardingAgent):
         )
 
         prompt = self.prompts["group_clusters"].format(
-            project_name=self.project_name,
             cfg_clusters=cluster_str,
-            meta_context=meta_context_str,
-            project_type=project_type,
         )
 
-        cluster_analysis = self._validation_invoke(
+        cluster_analysis = self._invoke_validate(
             prompt,
             ClusterAnalysis,
             validators=[validate_cluster_coverage],
-            context=ValidationContext(
+            validation_context=ValidationContext(
                 cluster_results=cluster_results,
                 expected_cluster_ids=get_all_cluster_ids(cluster_results),
             ),
@@ -108,20 +124,14 @@ class AbstractionAgent(ClusterMethodsMixin, CodeBoardingAgent):
     def step_final_analysis(
         self, llm_cluster_analysis: ClusterAnalysis, cluster_results: dict[str, ClusterResult]
     ) -> AnalysisInsights:
-        logger.info(f"[AbstractionAgent] Generating final analysis for: {self.project_name}")
-
-        meta_context_str = self.meta_context.llm_str() if self.meta_context else "No project context available."
-        project_type = self.meta_context.project_type if self.meta_context else "unknown"
+        logger.info(f"[AbstractionAgent] Generating final component analysis for: {self.project_name}")
 
         cluster_str = llm_cluster_analysis.llm_str() if llm_cluster_analysis else "No cluster analysis available."
 
         group_names = [cc.name for cc in llm_cluster_analysis.cluster_components] if llm_cluster_analysis else []
 
         prompt = self.prompts["final_analysis"].format(
-            project_name=self.project_name,
             cluster_analysis=cluster_str,
-            meta_context=meta_context_str,
-            project_type=project_type,
         )
 
         if group_names:
@@ -130,25 +140,80 @@ class AbstractionAgent(ClusterMethodsMixin, CodeBoardingAgent):
                 f"Every one of these names must appear in exactly one component's source_group_names: {group_names}\n"
             )
 
-        # Build validation context with CFG graphs for edge checking
         context = ValidationContext(
             cluster_results=cluster_results,
-            cfg_graphs={lang: self.static_analysis.get_cfg(lang) for lang in self.static_analysis.get_languages()},
             static_analysis=self.static_analysis,
             llm_cluster_analysis=llm_cluster_analysis,
         )
 
-        return self._validation_invoke(
+        architecture = self._invoke_repair_validate(
             prompt,
-            AnalysisInsights,
+            ComponentArchitecture,
+            repairs=[repair_component_group_names, repair_key_entities],
             validators=[
-                validate_relation_component_names,
                 validate_group_name_coverage,
                 validate_key_entities,
             ],
-            context=context,
+            repair_context=ComponentRepairContext(
+                reference_resolver=self.reference_resolver,
+                cluster_results=cluster_results,
+                llm_cluster_analysis=llm_cluster_analysis,
+            ),
+            validation_context=context,
+            max_validation_attempts=3,
+        )
+        return AnalysisInsights(
+            description=architecture.description,
+            components=architecture.components,
+            components_relations=[],
+        )
+
+    @trace
+    def step_api_surfaces(self, analysis: AnalysisInsights) -> ComponentApiSurfaces:
+        logger.info(f"[AbstractionAgent] Analyzing component API surfaces for: {self.project_name}")
+        static_call_evidence = self.build_scope_cfg_string(analysis)
+        prompt = self.prompts["api_surfaces"].format(
+            component_summaries=analysis.llm_str(),
+            static_call_evidence=static_call_evidence,
+        )
+        return self._parse_invoke(prompt, ComponentApiSurfaces)
+
+    @trace
+    def step_relation_analysis(
+        self,
+        analysis: AnalysisInsights,
+        api_surfaces: ComponentApiSurfaces,
+        cluster_analysis: ClusterAnalysis,
+        cluster_results: dict[str, ClusterResult],
+    ) -> None:
+        logger.info(f"[AbstractionAgent] Discovering component relations for: {self.project_name}")
+        static_call_evidence = self.build_scope_cfg_string(analysis)
+        cfg_graphs = self.static_analysis.available_cfgs()
+        self.toolkit.context.cluster_analysis = cluster_analysis
+        self.toolkit.context.cluster_results = cluster_results
+        self.toolkit.context.cfg_graphs = cfg_graphs
+        prompt = self.prompts["relation_analysis"].format(
+            component_summaries=analysis.llm_str(),
+            api_surfaces=api_surfaces.llm_str(),
+            static_call_evidence=static_call_evidence,
+        )
+        relation_result = self._invoke_validate(
+            prompt,
+            ComponentRelations,
+            validators=[validate_relations],
+            validation_context=ValidationContext(
+                cluster_results=cluster_results,
+                cfg_graphs=cfg_graphs,
+                repo_dir=str(self.repo_dir),
+                static_analysis=self.static_analysis,
+                llm_cluster_analysis=cluster_analysis,
+                components=analysis.components,
+            ),
             max_validation_attempts=3,
         )
+        analysis.components_relations = relation_result.components_relations
+        assign_relation_ids(analysis)
+        self.build_static_relations(analysis)
 
     def run(self):
         # Build full cluster results dict for all languages ONCE
@@ -166,12 +231,17 @@ class AbstractionAgent(ClusterMethodsMixin, CodeBoardingAgent):
         # Step 5: Populate file_methods deterministically from cluster results + orphan assignment
         self.populate_file_methods(analysis, cluster_results)
 
-        # Step 6: Build static inter-component relations from CFG edges
-        self.build_static_relations(analysis)
+        # Step 6: Analyze component API surfaces
+        api_surfaces = self.step_api_surfaces(analysis)
+
+        # Step 7: Discover relations from API surfaces and attach deterministic all_edges
+        self.step_relation_analysis(analysis, api_surfaces, cluster_analysis, cluster_results)
 
-        # Step 7: Fix source code reference lines (resolves reference_file paths for key_entities)
-        analysis = self.fix_source_code_reference_lines(analysis)
-        # Step 8: Ensure unique key entities across components
+        # Step 8: Fix source code reference lines (resolves reference_file paths for key_entities and key_edges)
+        analysis = self.reference_resolver.fix_source_code_reference_lines(analysis)
+        # Step 9: Index relation endpoints after reference resolution
+        index_relation_endpoints(analysis, self.repo_dir)
+        # Step 10: Ensure unique key entities across components
         self._ensure_unique_key_entities(analysis)
 
         return analysis, cluster_results

{codeboarding-0.12.4 → codeboarding-0.13.0}/agents/agent.py

@@ -1,6 +1,8 @@
 import json
 import logging
+from collections.abc import Callable
 from pathlib import Path
+from typing import Protocol, TypeVar
 
 from google.api_core.exceptions import ResourceExhausted
 from langchain_core.exceptions import OutputParserException
@@ -21,18 +23,43 @@ from agents.validation import ValidationResult, score_validation_results, VALIDA
 from monitoring.mixin import MonitoringMixin
 from repo_utils.ignore import RepoIgnoreManager
 from agents.agent_responses import LLMBaseModel
-from agents.llm_config import MONITORING_CALLBACK
+from agents.llm_config import MONITORING_CALLBACK, current_provider_key_context
+from agents.llm_errors import detect_auth_error
 from static_analyzer.analysis_result import StaticAnalysisResults
-from static_analyzer.reference_resolve_mixin import ReferenceResolverMixin
+from static_analyzer.reference_resolver import StaticReferenceResolver
 
 logger = logging.getLogger(__name__)
 
+ParseResultT = TypeVar("ParseResultT")
+ResultT = TypeVar("ResultT", bound="RepairValidationResult")
+RepairContextT = TypeVar("RepairContextT")
+ValidationContextT = TypeVar("ValidationContextT")
+
+
+class RepairValidationResult(Protocol):
+    def llm_str(self) -> str: ...
+
 
 class EmptyExtractorMessageError(ValueError):
     """Raised when extractor returns an empty message payload."""
 
 
-class CodeBoardingAgent(ReferenceResolverMixin, MonitoringMixin):
+def _raise_if_auth_error(exc: Exception) -> None:
+    """Re-raise *exc* as a typed :class:`LLMAuthError` when it's a rejected key.
+
+    Shared by the retry classifiers in ``_invoke`` and ``_parse_response``: a
+    401 is permanent for the run, so raising here propagates it terminally (the
+    retry loop never sees a RETRY decision) and the CLI can surface an
+    actionable message. Non-auth exceptions fall through untouched.
+    """
+    provider, key_tail = current_provider_key_context()
+    auth_error = detect_auth_error(exc, provider=provider, key_tail=key_tail)
+    if auth_error is not None:
+        logger.error("LLM auth failure — not retrying: %s", auth_error)
+        raise auth_error from exc
+
+
+class CodeBoardingAgent(MonitoringMixin):
     def __init__(
         self,
         repo_dir: Path,
@@ -41,11 +68,12 @@ class CodeBoardingAgent(ReferenceResolverMixin, MonitoringMixin):
         agent_llm: BaseChatModel,
         parsing_llm: BaseChatModel,
     ):
-        ReferenceResolverMixin.__init__(self, repo_dir, static_analysis)
         MonitoringMixin.__init__(self)
         self.parsing_llm = parsing_llm
         self.agent_llm = agent_llm
         self.repo_dir = repo_dir
+        self.static_analysis = static_analysis
+        self.reference_resolver = StaticReferenceResolver(repo_dir, static_analysis)
         self.ignore_manager = RepoIgnoreManager(repo_dir)
 
         context = RepoContext(repo_dir=repo_dir, ignore_manager=self.ignore_manager, static_analysis=static_analysis)
@@ -55,7 +83,6 @@ class CodeBoardingAgent(ReferenceResolverMixin, MonitoringMixin):
             model=agent_llm,
             tools=self.toolkit.get_agent_tools(),
         )
-        self.static_analysis = static_analysis
         self.system_message = SystemMessage(content=system_message)
 
     @property
@@ -82,6 +109,10 @@ class CodeBoardingAgent(ReferenceResolverMixin, MonitoringMixin):
     def read_method_invocations_tool(self):
         return self.toolkit.read_method_invocations
 
+    @property
+    def component_bridge_edges_tool(self):
+        return self.toolkit.component_bridge_edges
+
     @property
     def read_file_tool(self):
         return self.toolkit.read_file
@@ -132,6 +163,7 @@ class CodeBoardingAgent(ReferenceResolverMixin, MonitoringMixin):
             return ""  # unreachable for AIMessage but satisfies typing
 
         def classify(exc: Exception, attempt: int) -> RetryDecision:
+            _raise_if_auth_error(exc)
             if getattr(exc, "status_code", None) == 404:
                 logger.error(f"Permanent HTTP 404 — not retrying: {type(exc).__name__}: {exc}")
                 return RetryDecision(action=RetryAction.GIVE_UP)
@@ -201,12 +233,33 @@ class CodeBoardingAgent(ReferenceResolverMixin, MonitoringMixin):
         except Empty:
             raise RuntimeError("Agent invocation completed but no result was returned")
 
-    def _parse_invoke(self, prompt: str, type: type, include_hidden: bool = False):
+    def _parse_invoke(
+        self,
+        prompt: str,
+        return_type: type[ParseResultT],
+        include_hidden: bool = False,
+    ) -> ParseResultT:
         response = self._invoke(prompt)
         assert isinstance(response, str), f"Expected a string as response type got {response}"
-        return self._parse_response(prompt, response, type, include_hidden=include_hidden)
+        return self._parse_response(prompt, response, return_type, include_hidden=include_hidden)
 
-    def _score_result(self, result, validators: list, context) -> tuple[float, list[tuple[float, str]]]:
+    def _repair_result(
+        self,
+        result: ResultT,
+        repairs: list[Callable[[ResultT, RepairContextT], None]],
+        repair_context: RepairContextT,
+    ) -> None:
+        """Apply deterministic repairs to one parsed candidate."""
+        for repair in repairs:
+            logger.info("[Repair] Applying %s", repair.__name__)
+            repair(result, repair_context)
+
+    def _score_result(
+        self,
+        result: ResultT,
+        validators: list[Callable[[ResultT, ValidationContextT], ValidationResult]],
+        validation_context: ValidationContextT,
+    ) -> tuple[float, list[tuple[float, str]]]:
         """Run all validators on a result and return (score, prioritized_feedback).
 
         The score is computed using weighted validators where coverage-related
@@ -217,10 +270,10 @@ class CodeBoardingAgent(ReferenceResolverMixin, MonitoringMixin):
         weight descending, so that the LLM focuses on the most critical issues
         (cluster/group coverage) before lower-priority ones (key entities).
         """
-        validator_results: list[tuple] = []
+        validator_results: list[tuple[Callable[[ResultT, ValidationContextT], ValidationResult], ValidationResult]] = []
         weighted_feedback: list[tuple[float, str]] = []
         for validator in validators:
-            validator_result: ValidationResult = validator(result, context)
+            validator_result: ValidationResult = validator(result, validation_context)
             validator_results.append((validator, validator_result))
             if not validator_result.is_valid:
                 weight = VALIDATOR_WEIGHTS.get(validator.__name__, DEFAULT_VALIDATOR_WEIGHT)
@@ -233,44 +286,52 @@ class CodeBoardingAgent(ReferenceResolverMixin, MonitoringMixin):
         score = score_validation_results(validator_results)
         return score, weighted_feedback
 
-    def _validation_invoke(
+    def _invoke_repair_validate(
         self,
         prompt: str,
-        return_type: type,
-        validators: list,
-        context,
+        return_type: type[ResultT],
+        repairs: list[Callable[[ResultT, RepairContextT], None]],
+        validators: list[Callable[[ResultT, ValidationContextT], ValidationResult]],
+        repair_context: RepairContextT,
+        validation_context: ValidationContextT,
         max_validation_attempts: int = 1,
         include_hidden: bool = False,
-    ):
-        """
-        Invoke LLM with validation, feedback loop, and best-of-N selection.
-
-        Each attempt (initial + retries) is scored using weighted validators.
-        Coverage validators (validate_cluster_coverage, validate_group_name_coverage)
-        are weighted ~2x higher than other validators, so the selection strongly
-        favours results with complete coverage.
-
-        If any attempt scores perfectly (all validators pass), it is returned
-        immediately. Otherwise the highest-scoring result across all attempts is
-        returned.
-
-        Args:
-            prompt: The original prompt
-            return_type: Pydantic type to parse into
-            validators: List of validation functions to run
-            context: ValidationContext with data needed for validation
-            max_validation_attempts: Maximum validation attempts (initial attempt included).
-                Retries occur only when this value is greater than 1. (default: 1)
-
-        Returns:
-            The highest-scoring result of return_type across all attempts
-        """
+    ) -> ResultT:
+        """Bind deterministic repairs to every candidate before validation."""
+
+        def repair_candidate(result: ResultT) -> None:
+            self._repair_result(result, repairs, repair_context)
+
+        return self._invoke_validate(
+            prompt,
+            return_type,
+            validators,
+            validation_context,
+            max_validation_attempts=max_validation_attempts,
+            include_hidden=include_hidden,
+            candidate_repair=repair_candidate,
+        )
+
+    def _invoke_validate(
+        self,
+        prompt: str,
+        return_type: type[ResultT],
+        validators: list[Callable[[ResultT, ValidationContextT], ValidationResult]],
+        validation_context: ValidationContextT,
+        max_validation_attempts: int = 1,
+        include_hidden: bool = False,
+        candidate_repair: Callable[[ResultT], None] | None = None,
+    ) -> ResultT:
+        """Validate parsed candidates and return the best-scoring result."""
         # Compute the maximum possible score so we can detect a perfect result
         max_possible_score = sum(VALIDATOR_WEIGHTS.get(v.__name__, DEFAULT_VALIDATOR_WEIGHT) for v in validators)
 
         result = self._parse_invoke(prompt, return_type, include_hidden=include_hidden)
+        if candidate_repair is not None:
+            candidate_repair(result)
         logger.info(
-            "[Validation] Parsed %s: %s",
+            "[Validation] Parsed%s %s: %s",
+            " and repaired" if candidate_repair is not None else "",
             return_type.__name__,
             result.llm_str()[:500],
         )
@@ -283,7 +344,7 @@ class CodeBoardingAgent(ReferenceResolverMixin, MonitoringMixin):
         critical_threshold = 10.0
 
         for attempt in range(1, max_validation_attempts + 1):
-            score, weighted_feedback = self._score_result(result, validators, context)
+            score, weighted_feedback = self._score_result(result, validators, validation_context)
 
             logger.info(
                 f"[Validation] Attempt {attempt}/{max_validation_attempts} "
@@ -327,6 +388,8 @@ class CodeBoardingAgent(ReferenceResolverMixin, MonitoringMixin):
                 f"with {len(weighted_feedback)} feedback items"
             )
             result = self._parse_invoke(feedback_prompt, return_type, include_hidden=include_hidden)
+            if candidate_repair is not None:
+                candidate_repair(result)
 
         return best_result
 
@@ -355,6 +418,7 @@ class CodeBoardingAgent(ReferenceResolverMixin, MonitoringMixin):
             return self._extractor_parse(response, return_type, parser, include_hidden=include_hidden)
 
         def classify(exc: Exception, attempt: int) -> RetryDecision:
+            _raise_if_auth_error(exc)
             if isinstance(exc, ResourceExhausted):
                 return RetryDecision(
                     action=RetryAction.RETRY,