codeboarding 0.9.0__py3-none-any.whl

agents/validation.py

@@ -0,0 +1,383 @@
+"""Validation utilities for LLM agent outputs."""
+
+import logging
+import os
+from dataclasses import dataclass, field
+
+from agents.agent_responses import AnalysisInsights, ClusterAnalysis, ComponentFiles
+from repo_utils import normalize_path
+from static_analyzer.graph import CallGraph, ClusterResult
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ValidationContext:
+    """
+    This class is used to provide the necessary context for validating different LLM steps.
+    It encapsulates all relevant information required by validation routines to ensure that each step in the LLM pipeline
+    is checked against the expected criteria.
+    """
+
+    cluster_results: dict[str, ClusterResult] = field(default_factory=dict)
+    cfg_graphs: dict[str, CallGraph] = field(default_factory=dict)  # For edge checking
+    expected_cluster_ids: set[int] = field(default_factory=set)
+    expected_files: set[str] = field(default_factory=set)
+    valid_component_names: set[str] = field(default_factory=set)  # For file classification validation
+    repo_dir: str | None = None  # For path normalization
+
+
+@dataclass
+class ValidationResult:
+    """Result of a validation check."""
+
+    is_valid: bool
+    feedback_messages: list[str] = field(default_factory=list)
+
+
+def validate_cluster_coverage(result: ClusterAnalysis, context: ValidationContext) -> ValidationResult:
+    """
+    Validate that all expected clusters are represented in the ClusterAnalysis.
+
+    Args:
+        result: ClusterAnalysis containing cluster_components
+        context: ValidationContext with expected_cluster_ids
+
+    Returns:
+        ValidationResult with feedback for missing clusters
+    """
+    if not context.expected_cluster_ids:
+        logger.warning("[Validation] No expected cluster IDs provided for coverage validation")
+        return ValidationResult(is_valid=True)
+
+    # Extract all cluster IDs from the result
+    result_cluster_ids = set()
+    for component in result.cluster_components:
+        result_cluster_ids.update(component.cluster_ids)
+
+    # Find missing clusters
+    missing_clusters = context.expected_cluster_ids - result_cluster_ids
+
+    if not missing_clusters:
+        logger.info("[Validation] All clusters are represented in the ClusterAnalysis")
+        return ValidationResult(is_valid=True)
+
+    # Build feedback message
+    missing_str = ", ".join(str(cid) for cid in sorted(missing_clusters))
+    feedback = (
+        f"The following cluster IDs are missing from the analysis: {missing_str}. "
+        f"Please ensure all clusters are assigned to a component or create new components for them."
+    )
+
+    logger.warning(f"[Validation] Missing clusters: {missing_str}")
+    return ValidationResult(is_valid=False, feedback_messages=[feedback])
+
+
+def validate_component_relationships(result: AnalysisInsights, context: ValidationContext) -> ValidationResult:
+    """
+    Validate that component relationships have corresponding edges in the cluster graph.
+
+    Args:
+        result: AnalysisInsights containing components and components_relations
+        context: ValidationContext with cluster_results and cfg_graphs
+
+    Returns:
+        ValidationResult with feedback for invalid relationships
+    """
+    if not context.cfg_graphs or not result.components_relations:
+        logger.warning("[Validation] No CFG graphs or component relationships provided for relationship validation")
+        return ValidationResult(is_valid=True)
+
+    # Build component name -> source_cluster_ids mapping
+    component_clusters: dict[str, list[int]] = {}
+    for component in result.components:
+        component_clusters[component.name] = component.source_cluster_ids
+
+    cluster_edge_lookup = _build_cluster_edge_lookup(context.cluster_results, context.cfg_graphs)
+
+    invalid_relations: list[str] = []
+
+    for relation in result.components_relations:
+        src_clusters = component_clusters.get(relation.src_name, [])
+        dst_clusters = component_clusters.get(relation.dst_name, [])
+
+        if not src_clusters or not dst_clusters:
+            continue
+
+        # Check if any cluster pair has an edge
+        has_edge = _check_edge_between_cluster_sets(
+            src_clusters,
+            dst_clusters,
+            context.cluster_results,
+            context.cfg_graphs,
+            cluster_edge_lookup,
+        )
+
+        if not has_edge:
+            invalid_relations.append(f"({relation.src_name} -> {relation.dst_name})")
+
+    if not invalid_relations:
+        logger.info("[Validation] All component relationships have backing edges")
+        return ValidationResult(is_valid=True)
+
+    # Build feedback message
+    invalid_str = ", ".join(invalid_relations)
+    feedback = (
+        f"The following component relationships lack backing edges in the cluster graph: {invalid_str}. "
+        f"Please double-check if these components are actually related. If there is no direct edge between "
+        f"their clusters, the relationship may be indirect or incorrect."
+    )
+
+    logger.warning(f"[Validation] Invalid relationships: {invalid_str}")
+    return ValidationResult(is_valid=False, feedback_messages=[feedback])
+
+
+def validate_key_entities(result: AnalysisInsights, context: ValidationContext) -> ValidationResult:
+    """
+    Validate that every component in AnalysisInsights has at least one key_entity assigned.
+
+    Args:
+        result: AnalysisInsights containing components
+        context: ValidationContext (not used but kept for interface consistency)
+
+    Returns:
+        ValidationResult with feedback for components missing key entities
+    """
+    components_without_key_entities: list[str] = []
+
+    for component in result.components:
+        if not component.key_entities or len(component.key_entities) == 0:
+            components_without_key_entities.append(component.name)
+
+    if not components_without_key_entities:
+        logger.info("[Validation] All components have key entities assigned")
+        return ValidationResult(is_valid=True)
+
+    # Build feedback message
+    missing_str = ", ".join(components_without_key_entities)
+    feedback = (
+        f"The following components are missing key entities: {missing_str}. "
+        f"Every component must have at least one key entity (critical class or method) "
+        f"that represents its core functionality. Please identify and add 2-5 key entities "
+        f"for each component."
+    )
+
+    logger.warning(f"[Validation] Components without key entities: {missing_str}")
+    return ValidationResult(is_valid=False, feedback_messages=[feedback])
+
+
+def validate_cluster_ids_populated(result: AnalysisInsights, context: ValidationContext) -> ValidationResult:
+    """
+    Validate that every cluster is assigned to at least one component.
+
+    Args:
+        result: AnalysisInsights containing components
+        context: ValidationContext with cluster_results to get available cluster IDs
+
+    Returns:
+        ValidationResult with feedback for unassigned clusters
+    """
+    if not context.cluster_results:
+        logger.warning("[Validation] No cluster results provided for cluster ID validation")
+        return ValidationResult(is_valid=True)
+
+    all_cluster_ids: set[int] = set()
+    for lang_result in context.cluster_results.values():
+        all_cluster_ids.update(lang_result.get_cluster_ids())
+
+    if not all_cluster_ids:
+        logger.warning("[Validation] No cluster IDs available for cluster ID validation")
+        return ValidationResult(is_valid=True)
+
+    assigned_cluster_ids: set[int] = set()
+    for component in result.components:
+        assigned_cluster_ids.update(component.source_cluster_ids or [])
+
+    unassigned_clusters = all_cluster_ids - assigned_cluster_ids
+
+    if not unassigned_clusters:
+        logger.info("[Validation] All clusters are assigned to components")
+        return ValidationResult(is_valid=True)
+
+    missing_str = ", ".join(str(cid) for cid in sorted(unassigned_clusters))
+    feedback = (
+        f"The following cluster IDs are not assigned to any component: {missing_str}. "
+        f"Please assign every cluster to a component based on which code clusters belong to it."
+    )
+
+    logger.warning(f"[Validation] Unassigned clusters: {missing_str}")
+    return ValidationResult(is_valid=False, feedback_messages=[feedback])
+
+
+def validate_file_classifications(result: ComponentFiles, context: ValidationContext) -> ValidationResult:
+    """
+    Validate that all unassigned files were classified to valid component names.
+
+    This validator is used for _classify_unassigned_files_with_llm to ensure:
+    1. All input files are present in the result
+    2. All component names are valid (exist in valid_component_names)
+
+    Args:
+        result: ComponentFiles with file_paths containing FileClassification objects
+        context: ValidationContext with expected_files (unassigned files) and valid_component_names
+
+    Returns:
+        ValidationResult with feedback for missing files or invalid component names
+    """
+    if not context.expected_files:
+        logger.warning("[Validation] No expected files provided for file classification validation")
+        return ValidationResult(is_valid=True)
+
+    feedback_messages = []
+
+    # Get classified file paths from result
+    classified_files = {normalize_path(fc.file_path, context.repo_dir) for fc in result.file_paths}
+
+    # Normalize paths for comparison
+    expected_files_normalized = {normalize_path(file_path, context.repo_dir) for file_path in context.expected_files}
+
+    # Check 1: Are all unassigned files classified?
+    missing_files = expected_files_normalized - classified_files
+    if missing_files:
+        missing_list = sorted(str(f) for f in missing_files)[:10]
+        missing_str = ", ".join(missing_list)
+        more_msg = f" and {len(missing_files) - 10} more" if len(missing_files) > 10 else ""
+        feedback_messages.append(
+            f"The following files were not classified: {missing_str}{more_msg}. "
+            f"Please ensure all files are assigned to a component."
+        )
+
+    # Check 2: Are all component names valid?
+    if context.valid_component_names:
+        invalid_classifications = []
+        for fc in result.file_paths:
+            if fc.component_name not in context.valid_component_names:
+                invalid_classifications.append(f"{fc.file_path} -> {fc.component_name}")
+
+        if invalid_classifications:
+            invalid_str = ", ".join(invalid_classifications[:10])
+            more_msg = f" and {len(invalid_classifications) - 10} more" if len(invalid_classifications) > 10 else ""
+            valid_names = ", ".join(sorted(context.valid_component_names))
+            feedback_messages.append(
+                f"Invalid component names found: {invalid_str}{more_msg}. "
+                f"Valid component names are: {valid_names}. "
+                f"Please use only these component names."
+            )
+
+    if not feedback_messages:
+        logger.info("[Validation] All unassigned files correctly classified")
+        return ValidationResult(is_valid=True)
+
+    logger.warning(f"[Validation] File classification issues: {len(feedback_messages)} problems found")
+    return ValidationResult(is_valid=False, feedback_messages=feedback_messages)
+
+
+def validate_relation_component_names(result: AnalysisInsights, _context: ValidationContext) -> ValidationResult:
+    """
+    Validate that every src_name and dst_name in components_relations refers to an existing component.
+
+    When a relation references a component name that does not exist, assign_component_ids will
+    leave src_id or dst_id as an empty string, producing broken references in the output JSON.
+
+    Args:
+        result: AnalysisInsights containing components and components_relations
+        context: ValidationContext (not used but kept for interface consistency)
+
+    Returns:
+        ValidationResult with feedback listing every relation whose src_name or dst_name is unknown
+    """
+    known_names = {component.name for component in result.components}
+
+    invalid_relations: list[str] = []
+    for relation in result.components_relations:
+        unknown: list[str] = []
+        if relation.src_name not in known_names:
+            unknown.append(f"src_name='{relation.src_name}'")
+        if relation.dst_name not in known_names:
+            unknown.append(f"dst_name='{relation.dst_name}'")
+        if unknown:
+            invalid_relations.append(
+                f"({relation.src_name} -{relation.relation}-> {relation.dst_name}): {', '.join(unknown)}"
+            )
+
+    if not invalid_relations:
+        logger.info("[Validation] All relation component names refer to existing components")
+        return ValidationResult(is_valid=True)
+
+    invalid_str = "; ".join(invalid_relations)
+    known_str = ", ".join(sorted(known_names)) if known_names else "<none>"
+    feedback = (
+        f"The following relations reference component names that do not exist: {invalid_str}. "
+        f"Known component names are: {known_str}. "
+        f"Please ensure that src_name and dst_name in every relation match an existing component name exactly."
+    )
+
+    logger.warning(f"[Validation] Relations with unknown component names: {invalid_str}")
+    return ValidationResult(is_valid=False, feedback_messages=[feedback])
+
+
+def _build_cluster_edge_lookup(
+    cluster_results: dict[str, ClusterResult],
+    cfg_graphs: dict[str, CallGraph],
+) -> dict[str, set[tuple[int, int]]]:
+    """Build a lookup of (src_cluster_id, dst_cluster_id) edges per language."""
+    cluster_edge_lookup: dict[str, set[tuple[int, int]]] = {}
+
+    for lang, cfg in cfg_graphs.items():
+        cluster_result = cluster_results.get(lang)
+        if not cluster_result:
+            continue
+
+        node_to_cluster: dict[str, int] = {}
+        for cluster_id, nodes in cluster_result.clusters.items():
+            for node in nodes:
+                node_to_cluster[node] = cluster_id
+
+        cluster_edges: set[tuple[int, int]] = set()
+        for edge in cfg.edges:
+            src_cluster = node_to_cluster.get(edge.get_source())
+            dst_cluster = node_to_cluster.get(edge.get_destination())
+            if src_cluster is None or dst_cluster is None:
+                continue
+            cluster_edges.add((src_cluster, dst_cluster))
+
+        cluster_edge_lookup[lang] = cluster_edges
+
+    return cluster_edge_lookup
+
+
+def _check_edge_between_cluster_sets(
+    src_cluster_ids: list[int],
+    dst_cluster_ids: list[int],
+    cluster_results: dict[str, ClusterResult],
+    cfg_graphs: dict[str, CallGraph],
+    cluster_edge_lookup: dict[str, set[tuple[int, int]]] | None = None,
+) -> bool:
+    """
+    Check if there's an edge between any pair of clusters from two sets.
+
+    Args:
+        src_cluster_ids: Source cluster IDs
+        dst_cluster_ids: Destination cluster IDs
+        cluster_results: dict mapping language -> ClusterResult
+        cfg_graphs: dict mapping language -> CallGraph
+        cluster_edge_lookup: Optional precomputed (src_cluster, dst_cluster) edges per language
+
+    Returns:
+        True if any edge exists between the cluster sets
+    """
+    if not src_cluster_ids or not dst_cluster_ids:
+        return False
+
+    if cluster_edge_lookup is None:
+        cluster_edge_lookup = _build_cluster_edge_lookup(cluster_results, cfg_graphs)
+
+    src_set = set(src_cluster_ids)
+    dst_set = set(dst_cluster_ids)
+
+    for cluster_edges in cluster_edge_lookup.values():
+        for src_cluster, dst_cluster in cluster_edges:
+            if src_cluster in src_set and dst_cluster in dst_set:
+                return True
+
+    return False

caching/__init__.py

@@ -0,0 +1,4 @@
+from caching.cache import BaseCache
+from caching.meta_cache import MetaCache, MetaCacheRecord
+
+__all__ = ["BaseCache", "MetaCache", "MetaCacheRecord"]

caching/cache.py

@@ -0,0 +1,29 @@
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import TypeVar, Generic
+
+T = TypeVar("T")
+
+
+class BaseCache(ABC, Generic[T]):
+
+    def __init__(self, filename: str, cache_dir: Path):
+        self.cache_dir = cache_dir
+        self.file_path = self.cache_dir / filename
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+    @abstractmethod
+    def load(self) -> T | None:
+        pass
+
+    @abstractmethod
+    def store(self, data: T) -> None:
+        pass
+
+    @abstractmethod
+    def signature(self) -> str:
+        pass
+
+    @abstractmethod
+    def is_stale(self, data: T) -> bool:
+        pass

caching/meta_cache.py

@@ -0,0 +1,227 @@
+import hashlib
+import json
+import logging
+import sqlite3
+from collections.abc import Sequence
+from pathlib import Path
+
+from langchain_community.cache import SQLiteCache
+from langchain_core.language_models import BaseChatModel
+from langchain_core.outputs import Generation
+from pydantic import BaseModel
+
+from agents.agent_responses import MetaAnalysisInsights
+from agents.dependency_discovery import FileRole, discover_dependency_files
+from caching.cache import BaseCache
+from repo_utils import Repo, require_git_import
+from repo_utils.ignore import RepoIgnoreManager
+from utils import get_cache_dir
+
+logger = logging.getLogger(__name__)
+
+type JsonScalar = str | int | float | bool | None
+
+_README_PATTERNS: tuple[str, ...] = (
+    "README.md",
+    "README.rst",
+    "README.txt",
+    "README",
+    "readme.md",
+)
+
+_CACHE_WATCH_ROLES: frozenset[FileRole] = frozenset({FileRole.MANIFEST, FileRole.CONFIG})
+
+
+class MetaCacheRecord(BaseModel):
+    meta: MetaAnalysisInsights
+    base_commit: str
+    watch_files: list[str]
+    watch_state_hash: str | None = None
+
+
+class MetaCache(BaseCache[MetaCacheRecord]):
+    """SQLite-backed cache for MetaAgent analysis results.
+
+    Watches dependency manifests, config files, and root-level READMEs.
+    Keyed by a composite of project name, prompt version hash, and LLM
+    configuration so that any change to prompts or models automatically
+    produces a cache miss.
+    """
+
+    def __init__(
+        self,
+        repo_dir: Path,
+        ignore_manager: RepoIgnoreManager,
+        project_name: str,
+        agent_llm: BaseChatModel,
+        parsing_llm: BaseChatModel,
+        prompt_material: str,
+    ):
+        super().__init__("meta_agent_llm.sqlite", cache_dir=get_cache_dir(repo_dir))
+        self._repo_dir = repo_dir
+        self._ignore_manager = ignore_manager
+        self._prompt_key = self._build_prompt_key(project_name, prompt_material)
+        self._llm_key = self._build_llm_key(agent_llm, parsing_llm)
+
+    @staticmethod
+    def _llm_signature(llm: BaseChatModel) -> str:
+        model_id = None
+        for attr in ("model_name", "model", "model_id"):
+            value = getattr(llm, attr, None)
+            if isinstance(value, str) and value:
+                model_id = value
+                break
+
+        config: dict[str, JsonScalar] = {}
+        for attr in ("temperature", "max_tokens", "top_p", "timeout", "max_retries"):
+            value = getattr(llm, attr, None)
+            if isinstance(value, (str, int, float, bool)) or value is None:
+                config[attr] = value
+
+        payload = {
+            "provider": f"{type(llm).__module__}.{type(llm).__name__}",
+            "model_id": model_id or type(llm).__name__,
+            "config": config,
+        }
+        return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
+
+    def _build_prompt_key(self, project_name: str, prompt_material: str) -> str:
+        prompt_hash = hashlib.sha256(prompt_material.encode("utf-8")).hexdigest()
+        payload = {
+            "kind": "meta_agent_cache",
+            "project_name": project_name,
+            "prompt_version": prompt_hash,
+        }
+        return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
+
+    def _build_llm_key(self, agent_llm: BaseChatModel, parsing_llm: BaseChatModel) -> str:
+        payload = {
+            "kind": "meta_agent_llm_cache",
+            "agent": self._llm_signature(agent_llm),
+            "parser": self._llm_signature(parsing_llm),
+        }
+        return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
+
+    def signature(self) -> str:
+        """Return the composite cache key identifying this configuration."""
+        return self._prompt_key + "|" + self._llm_key
+
+    def _open_sqlite(self) -> SQLiteCache | None:
+        try:
+            self.cache_dir.mkdir(parents=True, exist_ok=True)
+            return SQLiteCache(database_path=str(self.file_path))
+        except (OSError, sqlite3.Error) as e:
+            logger.warning("Meta cache disabled: %s", e)
+            return None
+
+    def load(self) -> MetaCacheRecord | None:
+        cache = self._open_sqlite()
+        if cache is None:
+            return None
+        raw: Sequence[Generation] | None = cache.lookup(self._prompt_key, self._llm_key)
+        if raw is None:
+            return None
+        if len(raw) > 1:
+            logger.warning("Meta cache returned %d generations; using first", len(raw))
+        try:
+            return MetaCacheRecord.model_validate_json(raw[0].text)
+        except Exception:
+            return None
+
+    def store(self, data: MetaCacheRecord) -> None:
+        cache = self._open_sqlite()
+        if cache is None:
+            return
+        cache.clear()
+        cache.update(self._prompt_key, self._llm_key, [Generation(text=data.model_dump_json())])
+
+    def clear(self) -> None:
+        cache = self._open_sqlite()
+        if cache is not None:
+            cache.clear()
+
+    @require_git_import(default=[])
+    def discover_watch_files(self) -> list[str]:
+        """Return git-known files whose changes should invalidate this cache.
+
+        Includes dependency manifests and configs (not locks) and root-level
+        README files that the meta agent reads for project context.
+        """
+        try:
+            repo = Repo(self._repo_dir)
+            tracked_files = set(repo.git.ls_files().splitlines())
+            untracked_files = {
+                Path(path).as_posix()
+                for path in repo.untracked_files
+                if not self._ignore_manager.should_ignore(Path(path))
+            }
+            git_known_files = tracked_files | untracked_files
+        except Exception as e:
+            logger.warning("Unable to discover git file set for meta cache watch list: %s", e)
+            return []
+
+        watch: set[str] = set()
+
+        for discovered in discover_dependency_files(self._repo_dir, self._ignore_manager, roles=_CACHE_WATCH_ROLES):
+            relative_path = discovered.path.relative_to(self._repo_dir).as_posix()
+            if relative_path in git_known_files:
+                watch.add(relative_path)
+
+        for pattern in _README_PATTERNS:
+            if (self._repo_dir / pattern).is_file() and pattern in git_known_files:
+                watch.add(pattern)
+
+        return sorted(watch)
+
+    @staticmethod
+    def _fingerprint_file(path: Path) -> bytes | None:
+        try:
+            digest = hashlib.sha256()
+            with path.open("rb") as handle:
+                for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+                    digest.update(chunk)
+            return digest.digest()
+        except OSError:
+            return None
+
+    def compute_watch_state_hash(self, watch_files: Sequence[str]) -> str | None:
+        """Return a deterministic fingerprint for watched file contents."""
+        if not watch_files:
+            return None
+
+        digest = hashlib.sha256()
+        for relative_path in sorted(set(watch_files)):
+            file_digest = self._fingerprint_file(self._repo_dir / relative_path)
+            if file_digest is None:
+                logger.warning("Unable to fingerprint meta cache watch file: %s", relative_path)
+                return None
+            digest.update(relative_path.encode("utf-8"))
+            digest.update(b"\0")
+            digest.update(file_digest)
+            digest.update(b"\n")
+
+        return digest.hexdigest()
+
+    def is_stale(self, record: MetaCacheRecord) -> bool:
+        """Return True if watched file fingerprints differ from the cached record."""
+        if not record.watch_files:
+            return False
+
+        if not record.watch_state_hash:
+            logger.info("Meta cache record is missing watch-state fingerprint; recomputing once for migration")
+            return True
+
+        expected_watch_files = sorted(set(record.watch_files))
+        discovered_watch_files = self.discover_watch_files()
+        if discovered_watch_files:
+            normalized_discovered = sorted(set(discovered_watch_files))
+            if normalized_discovered != expected_watch_files:
+                logger.info("Meta cache watch-file set changed; recomputing metadata analysis")
+                return True
+            expected_watch_files = normalized_discovered
+
+        current_watch_hash = self.compute_watch_state_hash(expected_watch_files)
+        if current_watch_hash is None:
+            return True
+
+        return current_watch_hash != record.watch_state_hash