shannon-codebase-insight 0.4.0__py3-none-any.whl

shannon_insight/primitives/detector.py

@@ -0,0 +1,318 @@
+"""
+Anomaly detection using rigorous statistical methods.
+
+Implements:
+- Mahalanobis distance for multivariate outliers
+- Robust Z-scores (MAD-based)
+- Grubbs' test for single outliers
+- Statistical significance testing
+"""
+
+from typing import Dict, List, Tuple
+import numpy as np
+
+from ..models import Primitives
+from ..logging_config import get_logger
+from ..exceptions import InvalidConfigError, InsufficientDataError
+from ..math import Statistics, RobustStatistics
+
+logger = get_logger(__name__)
+
+
+class AnomalyDetector:
+    """
+    Detect anomalies using statistically rigorous methods.
+    """
+
+    MIN_FILES_FOR_MAHALANOBIS = 10
+    MIN_FILES_FOR_Z_SCORE = 5
+
+    def __init__(
+        self,
+        primitives: Dict[str, Primitives],
+        threshold: float = 1.5,
+        use_multivariate: bool = True,
+    ):
+        """
+        Initialize anomaly detector.
+
+        Args:
+            primitives: Dictionary mapping file paths to primitives
+            threshold: Detection threshold (default 1.5 for z-scores)
+            use_multivariate: Use Mahalanobis distance for multivariate detection
+
+        Raises:
+            InvalidConfigError: If threshold is invalid
+            InsufficientDataError: If too few files for analysis
+        """
+        if threshold <= 0 or threshold >= 10:
+            raise InvalidConfigError(
+                "threshold", threshold, "Threshold must be between 0.0 and 10.0"
+            )
+
+        if len(primitives) < 3:
+            raise InsufficientDataError(
+                "Too few files for reliable analysis", minimum_required=3
+            )
+
+        self.primitives = primitives
+        self.threshold = threshold
+        self.use_multivariate = use_multivariate and len(primitives) >= self.MIN_FILES_FOR_MAHALANOBIS
+
+        logger.debug(
+            f"Initialized AnomalyDetector with threshold={threshold}, "
+            f"multivariate={self.use_multivariate}, files={len(primitives)}"
+        )
+
+    def normalize(self) -> Dict[str, Primitives]:
+        """
+        Normalize all primitives using robust statistical methods.
+
+        For small samples (<10): Uses modified Z-scores (MAD-based)
+        For larger samples: Uses standard Z-scores
+
+        Returns:
+            Dictionary mapping file paths to normalized primitives
+        """
+        normalized = {}
+
+        if self.use_multivariate:
+            return self._normalize_multivariate(normalized)
+
+        # Extract each primitive into separate lists
+        entropy_vals = [p.structural_entropy for p in self.primitives.values()]
+        centrality_vals = [p.network_centrality for p in self.primitives.values()]
+        volatility_vals = [p.churn_volatility for p in self.primitives.values()]
+        coherence_vals = [p.semantic_coherence for p in self.primitives.values()]
+        load_vals = [p.cognitive_load for p in self.primitives.values()]
+
+        # Use robust z-scores for small samples
+        if len(self.primitives) < self.MIN_FILES_FOR_Z_SCORE:
+            entropy_z = RobustStatistics.modified_z_score(entropy_vals)
+            centrality_z = RobustStatistics.modified_z_score(centrality_vals)
+            volatility_z = RobustStatistics.modified_z_score(volatility_vals)
+            coherence_z = RobustStatistics.modified_z_score(coherence_vals)
+            load_z = RobustStatistics.modified_z_score(load_vals)
+        else:
+            # Use standard z-scores for larger samples
+            entropy_z = Statistics.z_scores(entropy_vals)
+            centrality_z = Statistics.z_scores(centrality_vals)
+            volatility_z = Statistics.z_scores(volatility_vals)
+            coherence_z = Statistics.z_scores(coherence_vals)
+            load_z = Statistics.z_scores(load_vals)
+
+        # Build normalized primitives
+        paths = list(self.primitives.keys())
+        for i, path in enumerate(paths):
+            normalized[path] = Primitives(
+                structural_entropy=entropy_z[i],
+                network_centrality=centrality_z[i],
+                churn_volatility=volatility_z[i],
+                semantic_coherence=coherence_z[i],
+                cognitive_load=load_z[i],
+            )
+
+        return normalized
+
+    def _normalize_multivariate(
+        self, normalized: Dict[str, Primitives]
+    ) -> Dict[str, Primitives]:
+        """
+        Normalize using Mahalanobis distance for multivariate analysis.
+
+        Considers correlations between primitives, not just individual values.
+        """
+        # Build feature matrix
+        paths = list(self.primitives.keys())
+        n = len(paths)
+
+        features = np.array(
+            [
+                [
+                    self.primitives[path].structural_entropy,
+                    self.primitives[path].network_centrality,
+                    self.primitives[path].churn_volatility,
+                    self.primitives[path].semantic_coherence,
+                    self.primitives[path].cognitive_load,
+                ]
+                for path in paths
+            ]
+        )
+
+        # Compute mean and covariance
+        mean_vec = np.mean(features, axis=0)
+        cov_matrix = np.cov(features, rowvar=False)
+
+        # Compute Mahalanobis distance for each file
+        mahalanobis_distances = []
+        for i in range(n):
+            dist = Statistics.mahalanobis_distance(features[i], mean_vec, cov_matrix)
+            mahalanobis_distances.append(dist)
+
+        # Convert distances to z-like scores
+        # MD² follows chi-squared distribution with k degrees of freedom
+        k = 5  # number of dimensions
+        from scipy import stats
+
+        # Convert MD² to p-values
+        p_values = [
+            1 - stats.chi2.cdf(dist, k) for dist in mahalanobis_distances
+        ]
+
+        # Convert to z-scores via inverse CDF.
+        # p ≈ 0 means maximally significant (large z), p ≈ 1 means no anomaly.
+        z_scores = []
+        for p in p_values:
+            if p <= 0:
+                z_scores.append(10.0)  # practical cap for extreme significance
+            elif p >= 1:
+                z_scores.append(0.0)
+            else:
+                z_scores.append(stats.norm.ppf(1 - p))
+
+        # Normalize each primitive separately for reporting
+        entropy_vals = features[:, 0]
+        centrality_vals = features[:, 1]
+        volatility_vals = features[:, 2]
+        coherence_vals = features[:, 3]
+        load_vals = features[:, 4]
+
+        entropy_z = Statistics.z_scores(entropy_vals.tolist())
+        centrality_z = Statistics.z_scores(centrality_vals.tolist())
+        volatility_z = Statistics.z_scores(volatility_vals.tolist())
+        coherence_z = Statistics.z_scores(coherence_vals.tolist())
+        load_z = Statistics.z_scores(load_vals.tolist())
+
+        # Build normalized primitives
+        for i, path in enumerate(paths):
+            normalized[path] = Primitives(
+                structural_entropy=entropy_z[i],
+                network_centrality=centrality_z[i],
+                churn_volatility=volatility_z[i],
+                semantic_coherence=coherence_z[i],
+                cognitive_load=load_z[i],
+            )
+
+        # Store Mahalanobis scores for later use
+        self.mahalanobis_scores = {paths[i]: z_scores[i] for i in range(n)}
+
+        return normalized
+
+    def detect_anomalies(
+        self, normalized: Dict[str, Primitives]
+    ) -> Dict[str, List[str]]:
+        """
+        Detect which primitives are anomalous.
+
+        For multivariate mode: Uses Mahalanobis distance
+        For univariate mode: Uses individual z-scores
+
+        Returns:
+            Dictionary mapping file paths to list of anomaly flags
+        """
+        anomalies = {}
+
+        if self.use_multivariate and hasattr(self, "mahalanobis_scores"):
+            return self._detect_multivariate_anomalies(normalized)
+
+        # Univariate detection
+        for path, prims in normalized.items():
+            flags = []
+
+            if abs(prims.structural_entropy) > self.threshold:
+                direction = "high" if prims.structural_entropy > 0 else "low"
+                flags.append(f"structural_entropy_{direction}")
+
+            if prims.network_centrality > self.threshold:
+                flags.append("high_centrality")
+
+            if abs(prims.churn_volatility) > self.threshold:
+                flags.append("high_volatility")
+
+            if abs(prims.semantic_coherence) > self.threshold:
+                direction = "low" if prims.semantic_coherence < 0 else "high"
+                flags.append(f"semantic_coherence_{direction}")
+
+            if prims.cognitive_load > self.threshold:
+                flags.append("high_cognitive_load")
+
+            if flags:
+                anomalies[path] = flags
+
+        return anomalies
+
+    def _detect_multivariate_anomalies(
+        self, normalized: Dict[str, Primitives]
+    ) -> Dict[str, List[str]]:
+        """
+        Detect anomalies using Mahalanobis distance.
+
+        Also identifies which specific primitives contribute to the anomaly.
+        """
+        anomalies = {}
+
+        # Use chi-squared critical value for significance
+        k = 5  # number of dimensions
+        from scipy import stats
+
+        critical_value = stats.chi2.ppf(0.95, k)  # 95% confidence
+
+        for path, mahalanobis_z in self.mahalanobis_scores.items():
+            if abs(mahalanobis_z) > self.threshold:
+                prims = normalized[path]
+                flags = []
+
+                # Identify which primitives are most anomalous.
+                # TODO: The 0.5 multiplier on threshold is a heuristic that
+                # relaxes per-primitive thresholds in multivariate mode.
+                # A rigorous alternative: decompose Mahalanobis D² into
+                # per-variable contributions via (x_i - mu_i)^2 * [Sigma^-1]_ii
+                # and compare each to its marginal chi-squared critical value.
+                if abs(prims.structural_entropy) > self.threshold * 0.5:
+                    direction = "high" if prims.structural_entropy > 0 else "low"
+                    flags.append(f"structural_entropy_{direction}")
+
+                if prims.network_centrality > self.threshold * 0.5:
+                    flags.append("high_centrality")
+
+                if abs(prims.churn_volatility) > self.threshold * 0.5:
+                    flags.append("high_volatility")
+
+                if abs(prims.semantic_coherence) > self.threshold * 0.5:
+                    direction = "low" if prims.semantic_coherence < 0 else "high"
+                    flags.append(f"semantic_coherence_{direction}")
+
+                if prims.cognitive_load > self.threshold * 0.5:
+                    flags.append("high_cognitive_load")
+
+                if flags:
+                    anomalies[path] = flags
+
+        return anomalies
+
+    def detect_outliers(
+        self, values: List[float], method: str = "grubbs"
+    ) -> List[int]:
+        """
+        Detect outliers in a univariate dataset.
+
+        Args:
+            values: List of values
+            method: Detection method ('grubbs', 'iqr', 'mad')
+
+        Returns:
+            List of outlier indices
+        """
+        if method == "grubbs":
+            result = Statistics.grubbs_test(values)
+            if result:
+                return [result[0]]
+            return []
+        elif method == "iqr":
+            outliers = RobustStatistics.iqr_outliers(values)
+            return [i for i, is_outlier in enumerate(outliers) if is_outlier]
+        elif method == "mad":
+            modified_z = RobustStatistics.modified_z_score(values)
+            return [i for i, z in enumerate(modified_z) if abs(z) > 3.5]
+        else:
+            raise ValueError(f"Unknown method: {method}")

shannon_insight/primitives/extractor.py

@@ -0,0 +1,278 @@
+"""Extract the 5 orthogonal quality primitives"""
+
+from pathlib import Path
+from collections import defaultdict
+from typing import Dict, List, Set, Optional
+from datetime import datetime
+import numpy as np
+
+from ..models import FileMetrics, Primitives
+from ..cache import AnalysisCache
+from ..logging_config import get_logger
+from ..math import Entropy, GraphMetrics, RobustStatistics as RobustStats
+
+logger = get_logger(__name__)
+
+
+class PrimitiveExtractor:
+    """Extract the 5 orthogonal quality primitives"""
+
+    def __init__(
+        self,
+        files: List[FileMetrics],
+        cache: Optional[AnalysisCache] = None,
+        config_hash: str = ""
+    ):
+        self.files = files
+        self.file_map = {f.path: f for f in files}
+        self.cache = cache
+        self.config_hash = config_hash
+        logger.debug(f"Initialized PrimitiveExtractor for {len(files)} files")
+
+    def extract_all(self) -> Dict[str, Primitives]:
+        """Extract all 5 primitives for each file"""
+        results = {}
+
+        # Build dependency graph (needed for centrality)
+        dep_graph = self._build_dependency_graph()
+
+        # Compute each primitive
+        entropies = self._compute_structural_entropy()
+        centralities = self._compute_network_centrality(dep_graph)
+        volatilities = self._compute_churn_volatility()
+        coherences = self._compute_semantic_coherence()
+        loads = self._compute_cognitive_load()
+
+        for file in self.files:
+            results[file.path] = Primitives(
+                structural_entropy=entropies.get(file.path, 0),
+                network_centrality=centralities.get(file.path, 0),
+                churn_volatility=volatilities.get(file.path, 0),
+                semantic_coherence=coherences.get(file.path, 0),
+                cognitive_load=loads.get(file.path, 0),
+            )
+
+        return results
+
+    # ---- Primitive 1: Structural Entropy ----
+
+    def _compute_structural_entropy(self) -> Dict[str, float]:
+        """Compute normalized entropy of AST node type distribution."""
+        entropies = {}
+
+        for file in self.files:
+            if not file.ast_node_types or sum(file.ast_node_types.values()) == 0:
+                entropies[file.path] = 0.0
+                continue
+
+            entropies[file.path] = Entropy.normalized(file.ast_node_types)
+
+        return entropies
+
+    # ---- Primitive 2: Network Centrality ----
+
+    # Standard library modules to ignore when building dependency graph
+    _STDLIB_NAMES = frozenset({
+        "abc", "ast", "asyncio", "base64", "bisect", "builtins", "calendar",
+        "cmath", "codecs", "collections", "concurrent", "contextlib", "copy",
+        "csv", "ctypes", "dataclasses", "datetime", "decimal", "difflib",
+        "email", "enum", "errno", "fcntl", "fileinput", "fnmatch", "fractions",
+        "ftplib", "functools", "gc", "getpass", "glob", "gzip", "hashlib",
+        "heapq", "hmac", "html", "http", "importlib", "inspect", "io",
+        "itertools", "json", "logging", "lzma", "math", "mimetypes",
+        "multiprocessing", "operator", "os", "pathlib", "pickle", "platform",
+        "pprint", "queue", "random", "re", "secrets", "select", "shelve",
+        "shlex", "shutil", "signal", "socket", "sqlite3", "ssl",
+        "statistics", "string", "struct", "subprocess", "sys", "tempfile",
+        "textwrap", "threading", "time", "timeit", "tkinter", "token",
+        "tomllib", "traceback", "types", "typing", "unicodedata", "unittest",
+        "urllib", "uuid", "venv", "warnings", "weakref", "xml", "zipfile",
+        "zlib",
+    })
+
+    # Common third-party packages to ignore
+    _THIRDPARTY_NAMES = frozenset({
+        "numpy", "np", "pandas", "pd", "scipy", "sklearn", "matplotlib",
+        "plt", "seaborn", "requests", "flask", "django", "fastapi",
+        "pydantic", "typer", "click", "rich", "diskcache", "pytest",
+        "setuptools", "wheel", "pip", "pkg_resources",
+    })
+
+    def _build_dependency_graph(self) -> Dict[str, Set[str]]:
+        """Build file dependency graph from internal project imports only."""
+        graph = defaultdict(set)
+
+        # Map import paths to actual files
+        file_by_name = {}
+        for file in self.files:
+            name = Path(file.path).stem
+            file_by_name[name] = file.path
+
+        skip_names = self._STDLIB_NAMES | self._THIRDPARTY_NAMES
+
+        for file in self.files:
+            for imp in file.imports:
+                # Extract the leaf module name
+                pkg_name = imp.split("/")[-1].split(".")[-1]
+
+                # Skip stdlib and third-party imports
+                if pkg_name in skip_names:
+                    continue
+
+                # Skip relative import markers
+                if pkg_name.startswith(".") or pkg_name == "":
+                    continue
+
+                # Only add edge if it points to a different file
+                if pkg_name in file_by_name and file_by_name[pkg_name] != file.path:
+                    graph[file.path].add(file_by_name[pkg_name])
+
+        return dict(graph)
+
+    def _compute_network_centrality(
+        self, graph: Dict[str, Set[str]]
+    ) -> Dict[str, float]:
+        """Compute PageRank centrality.
+
+        TODO: Delegate to GraphMetrics.pagerank() for consistency and
+        correctness. The inline implementation differs from the canonical
+        version: it uses (1-d) instead of (1-d)/N for base probability,
+        has no dangling-node redistribution, no convergence check, and
+        misses nodes that appear only as targets. The min-max normalization
+        at the end masks the base-probability difference but the other
+        issues remain.
+        """
+        # Initialize PageRank scores
+        scores = {f.path: 1.0 for f in self.files}
+        damping = 0.85
+        iterations = 20
+
+        # Build reverse graph (incoming edges)
+        incoming = defaultdict(set)
+        for src, targets in graph.items():
+            for tgt in targets:
+                incoming[tgt].add(src)
+
+        # PageRank iteration
+        for _ in range(iterations):
+            new_scores = {}
+
+            for file in self.files:
+                # Base probability
+                rank = 1 - damping
+
+                # Add contribution from incoming edges
+                for src in incoming.get(file.path, []):
+                    out_degree = len(graph.get(src, []))
+                    if out_degree > 0:
+                        rank += damping * (scores[src] / out_degree)
+
+                new_scores[file.path] = rank
+
+            scores = new_scores
+
+        # Normalize to [0, 1]
+        if scores:
+            max_score = max(scores.values())
+            if max_score > 0:
+                scores = {k: v / max_score for k, v in scores.items()}
+
+        return scores
+
+    # ---- Primitive 3: Churn Volatility ----
+
+    def _compute_churn_volatility(self) -> Dict[str, float]:
+        """Compute volatility of file modifications (filesystem-based)"""
+        volatilities = {}
+
+        # Since no git history, use file modification time as proxy
+        now = datetime.now().timestamp()
+        ages = [now - f.last_modified for f in self.files]
+
+        if not ages:
+            return {}
+
+        # Normalize age to volatility score
+        # Recent changes = high volatility
+        max_age = max(ages)
+
+        for file in self.files:
+            age = now - file.last_modified
+            # Invert: older = more stable = lower volatility
+            volatility = 1 - (age / max_age) if max_age > 0 else 0
+            volatilities[file.path] = volatility
+
+        return volatilities
+
+    # ---- Primitive 4: Semantic Coherence ----
+
+    def _compute_semantic_coherence(self) -> Dict[str, float]:
+        """Compute semantic coherence via TF-IDF clustering"""
+        from sklearn.feature_extraction.text import TfidfVectorizer
+        from sklearn.metrics.pairwise import cosine_similarity
+
+        # Build document corpus (use imports + exports)
+        documents = []
+        paths = []
+
+        for file in self.files:
+            tokens = file.imports + file.exports
+            doc = " ".join(tokens) if tokens else "empty"
+            documents.append(doc)
+            paths.append(file.path)
+
+        if len(documents) < 2:
+            return {f.path: 1.0 for f in self.files}
+
+        # Compute TF-IDF vectors
+        vectorizer = TfidfVectorizer(min_df=1, max_df=0.8)
+        try:
+            tfidf_matrix = vectorizer.fit_transform(documents)
+        except Exception as e:
+            logger.warning(f"TF-IDF vectorization failed: {e}")
+            return {f.path: 1.0 for f in self.files}
+
+        # Compute pairwise similarities
+        similarities = cosine_similarity(tfidf_matrix)
+
+        coherences = {}
+        n = len(paths)
+        for i, path in enumerate(paths):
+            # Coherence = average similarity to OTHER files (exclude self)
+            if n > 1:
+                other_sims = [similarities[i][j] for j in range(n) if j != i]
+                coherences[path] = float(np.mean(other_sims))
+            else:
+                coherences[path] = 1.0
+
+        return coherences
+
+    # ---- Primitive 5: Cognitive Load ----
+
+    def _compute_cognitive_load(self) -> Dict[str, float]:
+        """Compute cognitive load = concepts × complexity.
+
+        TODO: The formula CL = concepts * complexity * (1 + depth/10) is a
+        hand-tuned heuristic with no academic citation. The /10 divisor
+        maps typical nesting depths (0-5) to a 0-50% multiplier. Consider
+        citing Cant et al. (1995) "A Conceptual Complexity Metric" or
+        replacing with a validated cognitive complexity model such as
+        SonarSource's Cognitive Complexity (2017).
+        """
+        loads = {}
+
+        for file in self.files:
+            # Concepts = functions + structs + interfaces
+            concepts = file.functions + file.structs + file.interfaces
+
+            # Cognitive load = concepts × complexity × nesting
+            load = concepts * file.complexity_score * (1 + file.nesting_depth / 10)
+            loads[file.path] = load
+
+        # Normalize to [0, 1]
+        if loads:
+            max_load = max(loads.values())
+            if max_load > 0:
+                loads = {k: v / max_load for k, v in loads.items()}
+
+        return loads