codebase-stats 0.0.1__py3-none-any.whl

codebase_stats/__init__.py

@@ -0,0 +1,148 @@
+"""Codebase Statistics Analysis Library
+
+A comprehensive library for analyzing Python codebase metrics including coverage,
+test duration, code complexity, maintainability, and more.
+
+Main Modules:
+    - core: Data loading and preprocessing
+    - coverage: Coverage analysis and reporting
+    - metrics: Code quality metrics (complexity, maintainability, etc.)
+    - duration: Test duration analysis
+    - sizes: File size distribution analysis
+    - tree: Directory structure analysis
+    - lowcov: Low-coverage file identification and prioritization
+    - utils: Utility functions for formatting and analysis
+
+Example Usage:
+    from codebase_stats import load_coverage, precompute_coverage_stats
+    from codebase_stats import show_coverage_histogram
+    
+    # Load and analyze coverage data
+    data = load_coverage("coverage.json")
+    stats = precompute_coverage_stats(data)
+    
+    # Display coverage report
+    show_coverage_histogram(stats, bins=10, show_blame=True)
+"""
+
+# Core data functions
+from .core import (
+    load_coverage,
+    load_report,
+    precompute_coverage_stats,
+    extract_layer,
+    build_suffix_index,
+    suffix_lookup,
+    LAYER_MAP,
+    LAYER_ORDER,
+)
+
+# Coverage analysis
+from .coverage import (
+    show_coverage_histogram,
+    scan_pragma_counts,
+    scan_pragma_intervals,
+    show_pragma_histogram,
+)
+
+# Code quality metrics
+from .metrics import (
+    run_radon,
+    run_radon_mi,
+    run_radon_raw,
+    run_radon_hal,
+    show_complexity_histogram,
+    show_mi_histogram,
+    show_raw_histogram,
+    show_hal_histogram,
+    cc_rank,
+    mi_rank,
+)
+
+# Test duration
+from .duration import (
+    show_duration_histogram,
+    test_duration,
+)
+
+# File sizes
+from .sizes import (
+    show_file_size_distribution,
+)
+
+# Tree analysis
+from .tree import (
+    analyze_tree,
+)
+
+# Low coverage reporting
+from .lowcov import (
+    show_low_coverage,
+    parse_sorts,
+    priority_score,
+    VALID_SORT_FIELDS,
+)
+
+# Utilities
+from .utils import (
+    percentile,
+    format_line_ranges,
+    ascii_histogram,
+    blame_header,
+    fmt_seconds,
+)
+
+# Reporter
+from .reporter import (
+    CodebaseStatsReporter,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # Core
+    "load_coverage",
+    "load_report",
+    "precompute_coverage_stats",
+    "extract_layer",
+    "build_suffix_index",
+    "suffix_lookup",
+    "LAYER_MAP",
+    "LAYER_ORDER",
+    # Coverage
+    "show_coverage_histogram",
+    "scan_pragma_counts",
+    "scan_pragma_intervals",
+    "show_pragma_histogram",
+    # Metrics
+    "run_radon",
+    "run_radon_mi",
+    "run_radon_raw",
+    "run_radon_hal",
+    "show_complexity_histogram",
+    "show_mi_histogram",
+    "show_raw_histogram",
+    "show_hal_histogram",
+    "cc_rank",
+    "mi_rank",
+    # Duration
+    "show_duration_histogram",
+    "test_duration",
+    # Sizes
+    "show_file_size_distribution",
+    # Tree
+    "analyze_tree",
+    # Low coverage
+    "show_low_coverage",
+    "parse_sorts",
+    "priority_score",
+    "VALID_SORT_FIELDS",
+    # Utils
+    "percentile",
+    "format_line_ranges",
+    "ascii_histogram",
+    "blame_header",
+    "fmt_seconds",
+    # Reporter
+    "CodebaseStatsReporter",
+]

codebase_stats/core.py

@@ -0,0 +1,240 @@
+"""Core data loading and preprocessing functions."""
+
+import json
+import sys
+from pathlib import Path
+
+
+def load_coverage(path: str) -> dict:
+    """Load and parse a coverage.json file.
+    
+    Args:
+        path: Path to coverage.json file
+        
+    Returns:
+        Parsed coverage data dictionary
+        
+    Raises:
+        SystemExit: On file not found or invalid JSON
+    """
+    try:
+        with open(path, encoding="utf-8") as f:
+            data = json.load(f)
+        if "files" not in data:
+            raise KeyError("coverage JSON missing 'files' key")
+        return data
+    except FileNotFoundError:
+        sys.exit(f"❌  File not found: {path}")
+    except json.JSONDecodeError:
+        sys.exit(f"❌  Invalid JSON: {path}")
+
+
+def load_report(path: str) -> dict:
+    """Load and parse a pytest-json-report file.
+    
+    Args:
+        path: Path to pytest JSON report file
+        
+    Returns:
+        Parsed report data dictionary
+        
+    Raises:
+        SystemExit: On file not found or invalid JSON
+    """
+    try:
+        with open(path, encoding="utf-8") as f:
+            data = json.load(f)
+        if "tests" not in data:
+            sys.exit(f"❌  '{path}' has no 'tests' key — is this a pytest-json-report file?")
+        return data
+    except FileNotFoundError:
+        sys.exit(f"❌  File not found: {path}")
+    except json.JSONDecodeError:
+        sys.exit(f"❌  Invalid JSON: {path}")
+
+
+# Layer mapping and ordering constants
+LAYER_MAP = {
+    "domain": "Domain",
+    "application": "Application",
+    "infrastructure": "Infrastructure",
+    "services": "Services",
+    "repositories": "Repositories",
+    "use_cases": "Use Cases",
+    "api": "API",
+}
+
+LAYER_ORDER = {
+    "Domain": 1,
+    "Application": 2,
+    "Services": 3,
+    "Use Cases": 4,
+    "Repositories": 5,
+    "Infrastructure": 6,
+    "API": 7,
+    "Other": 8,
+}
+
+
+def extract_layer(path: str) -> str:
+    """Extract the architectural layer from a file path.
+    
+    Args:
+        path: File path to analyze
+        
+    Returns:
+        Layer name (e.g., "Domain", "Application", "Other")
+    """
+    parts = [p.lower() for p in Path(path).parts]
+    for key, name in LAYER_MAP.items():
+        if key in parts:
+            return name
+    return "Other"
+
+
+def build_suffix_index(radon_map: dict) -> dict:
+    """Build a suffix-keyed index for path matching.
+    
+    This allows matching coverage.json paths (which may use different root 
+    prefixes or be absolute vs relative) to radon output.
+    
+    Strategy: for each radon key, store it under every possible suffix:
+      '/home/user/proj/app/foo/bar.py'  →  keys: 'bar.py', 'foo/bar.py',
+                                            'app/foo/bar.py', ...
+    Then look up a coverage path by trying progressively longer suffixes
+    until one hits. The longest match wins to avoid false collisions.
+    
+    Args:
+        radon_map: Dictionary mapping file paths to radon analysis data
+        
+    Returns:
+        Dictionary mapping suffixes to radon values
+    """
+    index: dict[str, object] = {}  # suffix_str → radon value
+    for k, v in radon_map.items():
+        parts = Path(k).parts
+        for i in range(len(parts)):
+            suffix = "/".join(parts[i:])
+            # First writer wins for each suffix key (shortest path = most specific)
+            if suffix not in index:
+                index[suffix] = v
+    return index
+
+
+def suffix_lookup(index: dict, coverage_path: str):
+    """Look up a coverage.json path in a suffix index.
+    
+    Try suffixes from most-specific (full path) down to basename.
+    
+    Args:
+        index: Index built by build_suffix_index()
+        coverage_path: Path from coverage.json to look up
+        
+    Returns:
+        The radon value if found, None otherwise
+    """
+    parts = Path(coverage_path).parts
+    for i in range(len(parts)):
+        key = "/".join(parts[i:])
+        if key in index:
+            return index[key]
+    return None
+
+
+def precompute_coverage_stats(
+    data: dict,
+    complexity_map: dict = None,
+    mi_map: dict = None,
+    raw_map: dict = None,
+    hal_map: dict = None,
+) -> dict:
+    """Precompute statistics from coverage.json and radon data.
+    
+    One pass over data['files']. Returns everything both histogram and
+    list functions need so neither has to re-iterate the JSON.
+    
+    Args:
+        data: Parsed coverage.json data
+        complexity_map: Radon cyclomatic complexity data (optional)
+        mi_map: Radon maintainability index data (optional)
+        raw_map: Radon raw metrics data (optional)
+        hal_map: Radon Halstead metrics data (optional)
+        
+    Returns:
+        Dictionary with precomputed statistics including file_stats, project totals,
+        and coverage percentiles
+    """
+    proj_total = proj_covered = 0
+    t = data.get("totals", {})
+    proj_total = t.get("num_statements", 0)
+    proj_covered = t.get("covered_lines", 0)
+
+    # Build suffix indexes once — O(n·depth) — so the per-file lookup is O(depth)
+    _cc_idx = build_suffix_index(complexity_map) if complexity_map else {}
+    _mi_idx = build_suffix_index(mi_map) if mi_map else {}
+    _raw_idx = build_suffix_index(raw_map) if raw_map else {}
+    _hal_idx = build_suffix_index(hal_map) if hal_map else {}
+
+    file_stats = []  # one dict per file with non-zero statements
+    for path, info in data["files"].items():
+        s = info.get("summary", {})
+        total = s.get("num_statements", 0)
+        if total == 0:
+            continue
+        covered = s.get("covered_lines", 0)
+        pct = s.get("percent_covered", 0.0)
+        missing = info.get("missing_lines") or []
+        layer = extract_layer(path)
+
+        if proj_total == 0:  # totals block missing — accumulate
+            proj_total += total
+            proj_covered += covered
+
+        cc = suffix_lookup(_cc_idx, path) if _cc_idx else None
+        mi = suffix_lookup(_mi_idx, path) if _mi_idx else None
+        raw = suffix_lookup(_raw_idx, path) if _raw_idx else None
+        hal = suffix_lookup(_hal_idx, path) if _hal_idx else None
+        file_stats.append(
+            {
+                "path": path,
+                "pct": pct,
+                "total": total,
+                "covered": covered,
+                "missing_count": len(missing),
+                "missing_lines": missing,
+                "layer": layer,
+                "layer_order": LAYER_ORDER.get(layer, 8),
+                # radon cc
+                "cc_avg": cc["avg"] if cc else None,
+                "cc_max": cc["max"] if cc else None,
+                "cc_n_blocks": cc["n_blocks"] if cc else None,
+                "cc_scores": cc["scores"] if cc else None,  # individual function scores
+                # radon mi
+                "mi": mi["mi"] if mi else None,
+                "mi_rank": mi["rank"] if mi else None,
+                # radon raw
+                "comment_ratio": raw["comment_ratio"] if raw else None,
+                "sloc": raw["sloc"] if raw else None,
+                # radon hal
+                "hal_bugs": hal["bugs"] if hal else None,
+                "hal_difficulty": hal["difficulty"] if hal else None,
+            }
+        )
+
+    proj_pct = proj_covered / proj_total * 100 if proj_total else 0.0
+    coverages_sorted = sorted(f["pct"] for f in file_stats)
+
+    # Flat list of (cc_score, filepath) for all functions across all files
+    # used by the CC histogram to plot true function distribution.
+    cc_scores = sorted(
+        (score, f["path"]) for f in file_stats if f.get("cc_scores") for score in f["cc_scores"]
+    )
+
+    return {
+        "file_stats": file_stats,
+        "proj_total": proj_total,
+        "proj_covered": proj_covered,
+        "proj_pct": proj_pct,
+        "coverages_sorted": coverages_sorted,
+        "cc_scores": cc_scores,  # (cc_value, filepath) per function
+    }

codebase_stats/coverage.py

@@ -0,0 +1,295 @@
+"""Coverage analysis functions."""
+
+from pathlib import Path
+from .utils import percentile, format_line_ranges, ascii_histogram, blame_header
+
+
+def scan_pragma_intervals(root: str) -> dict[str, list]:
+    """Find sequential # pragma: no cover blocks per .py file under root.
+    
+    Args:
+        root: Root directory to scan
+        
+    Returns:
+        Dictionary mapping file paths to lists of (start, end) tuples
+    """
+    result = {}
+    for path in Path(root).rglob("*.py"):
+        if any(part.startswith(".") or part == "__pycache__" for part in path.parts):
+            continue
+        try:
+            lines = path.read_text(encoding="utf-8", errors="ignore").splitlines()
+            pragma_rows = [i + 1 for i, line in enumerate(lines) if "# pragma: no cover" in line]
+            if not pragma_rows:
+                continue
+            # Find sequential blocks
+            intervals = []
+            start = None
+            for idx, row in enumerate(pragma_rows):
+                if start is None:
+                    start = row
+                if idx == len(pragma_rows) - 1 or pragma_rows[idx + 1] != row + 1:
+                    end = row
+                    intervals.append((start, end))
+                    start = None
+            result[str(path)] = intervals
+        except OSError:
+            pass
+    return result
+
+
+def scan_pragma_counts(root: str) -> dict[str, int]:
+    """Count '# pragma: no cover' occurrences per .py file under root.
+    
+    Args:
+        root: Root directory to scan
+        
+    Returns:
+        Dictionary mapping file paths to pragma counts
+    """
+    result = {}
+    for path in Path(root).rglob("*.py"):
+        if any(part.startswith(".") or part == "__pycache__" for part in path.parts):
+            continue
+        try:
+            count = path.read_text(encoding="utf-8", errors="ignore").count("# pragma: no cover")
+            if count > 0:
+                result[str(path)] = count
+        except OSError:
+            pass
+    return result
+
+
+def show_pragma_histogram(pragma_counts: dict, bins: int = 10, blame_limit: int = 20, width: int = 80) -> None:
+    """Histogram of '# pragma: no cover' count per file + all-files listing.
+
+    Also prints a secondary histogram and table for sequential blocks of
+    pragmas (intervals) so that you can spot long runs of skipped code.
+    
+    Args:
+        pragma_counts: Dictionary mapping files to pragma counts
+        bins: Number of histogram bins
+        blame_limit: Maximum blamed items to display
+        width: Line width for output
+    """
+    # --- Sequential pragma interval histogram ---
+    pragma_intervals = scan_pragma_intervals(".")
+    interval_lengths = [
+        end - start + 1 for intervals in pragma_intervals.values() for (start, end) in intervals
+    ]
+    if interval_lengths:
+        n_int = len(interval_lengths)
+        total_int = sum(interval_lengths)
+        int_max = max(interval_lengths)
+        int_step = max((int_max + 0.1) / bins, 1)
+        edges = [i * int_step for i in range(bins + 1)]
+        int_buckets = [0] * bins
+        for v in interval_lengths:
+            placed = False
+            for bi in range(bins - 1):
+                if edges[bi] <= v < edges[bi + 1]:
+                    int_buckets[bi] += 1
+                    placed = True
+                    break
+            if not placed:
+                int_buckets[-1] += 1
+        int_labels = [f"{int(edges[i]):>4}\u2013{int(edges[i + 1]):<4} rows" for i in range(bins)]
+        print(f"\n{'═' * width}")
+        print(f"  PRAGMA INTERVAL LENGTH HISTOGRAM  (sequential blocks, {n_int} intervals)")
+        print(f"  [linear scale, {bins} bins]")
+        print()
+        ascii_histogram(int_buckets, int_labels, width=width)
+        print()
+        print(
+            f"  p50: {int(percentile(interval_lengths, 50)):<6} "
+            f"p75: {int(percentile(interval_lengths, 75)):<6} "
+            f"p90: {int(percentile(interval_lengths, 90)):<6} "
+            f"p99: {int(percentile(interval_lengths, 99)):<6}"
+        )
+        print(f"  avg: {total_int / n_int:<6.1f} max: {int_max:<6} total rows: {total_int}")
+        print(f"\n{'═' * width}")
+        print(f"  PRAGMA INTERVAL ROWS  {n_int}")
+        print(f"\n{'═' * width}")
+        for path, intervals in pragma_intervals.items():
+            for start, end in intervals:
+                print(f"  {path}  rows {start}-{end} ({end - start + 1} rows)")
+    # --- End interval histogram ---
+    if not pragma_counts:
+        return
+    file_list = sorted(pragma_counts.items(), key=lambda x: x[1], reverse=True)
+    counts_sorted = sorted(pragma_counts.values())
+    n = len(counts_sorted)
+    total = sum(counts_sorted)
+    m_max = counts_sorted[-1]
+
+    m_step = max((m_max + 0.1) / bins, 1)
+    edges = [i * m_step for i in range(bins + 1)]
+    p_buckets = [0] * bins
+    for v in counts_sorted:
+        placed = False
+        for bi in range(bins - 1):
+            if edges[bi] <= v < edges[bi + 1]:
+                p_buckets[bi] += 1
+                placed = True
+                break
+        if not placed:
+            p_buckets[-1] += 1
+    p_labels = [f"{int(edges[i]):>4}\u2013{int(edges[i + 1]):<4} pragmas" for i in range(bins)]
+
+    print(f"\n{'═' * width}")
+    print(f"  PRAGMA: NO COVER DISTRIBUTION  (files with \u22651 pragma: {n}, total: {total})")
+    print(f"  [linear scale, {bins} bins]")
+    print()
+    ascii_histogram(p_buckets, p_labels, width=width)
+    print()
+    print(
+        f"  p50: {int(percentile(counts_sorted, 50)):<6} "
+        f"p75: {int(percentile(counts_sorted, 75)):<6} "
+        f"p90: {int(percentile(counts_sorted, 90)):<6} "
+        f"p99: {int(percentile(counts_sorted, 99)):<6}"
+    )
+    print(f"  avg: {total / n:<6.1f} max: {m_max:<6} total pragmas: {total}")
+
+    shown = file_list if not blame_limit else file_list[:blame_limit]
+    tail = (
+        f"  (showing top {blame_limit}, --blame-limit 0 for all)"
+        if blame_limit and len(file_list) > blame_limit
+        else ""
+    )
+    print(f"\n{'═' * width}")
+    print(f"  PRAGMA FILES  {len(file_list)}{tail}")
+    print(f"\n{'═' * width}")
+    for path, count in shown:
+        print(f"  {count:>4} pragmas  {path}")
+
+
+def show_coverage_histogram(
+    stats: dict,
+    bins: int = 10,
+    blame_limit: int = 20,
+    show_blame: bool = True,
+    threshold: float = 90.0,
+    pragma_counts: dict | None = None,
+    width: int = 80,
+):
+    """Display coverage distribution histogram and low-coverage blame.
+    
+    Args:
+        stats: Precomputed statistics from precompute_coverage_stats()
+        bins: Number of histogram bins
+        blame_limit: Maximum blamed files to display
+        show_blame: Whether to show quality blame sections
+        threshold: Blame ceiling (show files below Q1 or this threshold, whichever is lower)
+        pragma_counts: Optional pragma count data to display
+        width: Line width for output
+    """
+    cs = stats["coverages_sorted"]
+    if not cs:
+        print("No files found.")
+        return
+
+    n = len(cs)
+    avg = sum(cs) / n
+    below_80 = sum(1 for c in cs if c < 80)
+    below_60 = sum(1 for c in cs if c < 60)
+
+    step = 100 / bins
+    buckets = [0] * bins
+    labels = []
+    for i in range(bins):
+        lo, hi = i * step, (i + 1) * step
+        labels.append(f"{lo:>5.1f}–{hi:<5.1f}%")
+        for c in cs:
+            if lo <= c < hi or (i == bins - 1 and c == 100.0):
+                buckets[i] += 1
+
+    print(f"\n{'═' * width}")
+    print("  COVERAGE DISTRIBUTION HISTOGRAM")
+    print(f"{'═' * width}")
+    print(f"  Files: {n}   Project coverage: {stats['proj_pct']:.1f}%")
+    print()
+    ascii_histogram(buckets, labels, width=width)
+    print()
+    print(f"{'─' * width}")
+    print("  PERCENTILES")
+    print(f"{'─' * width}")
+    for pct in (25, 50, 75, 90, 95, 99):
+        label = {25: "Q1", 50: "Q2/med", 75: "Q3"}.get(pct, f"p{pct}")
+        print(f"  {label:<8} {percentile(cs, pct):>6.1f}%")
+    print(f"  {'avg':<8} {avg:>6.1f}%")
+    print(f"  {'min':<8} {cs[0]:>6.1f}%")
+    print(f"  {'max':<8} {cs[-1]:>6.1f}%")
+    not_100 = sum(1 for c in cs if c < 100)
+    print(f"  {'not-100%':<8} {not_100} files ({not_100 / n * 100:.1f}%)")
+    print(f"  <80%     {below_80} files ({below_80 / n * 100:.1f}%)")
+    print(f"  <60%     {below_60} files ({below_60 / n * 100:.1f}%)")
+
+    # ── missing lines histogram (non-100% files only) ──────────────────────
+    missing_counts = sorted(f["missing_count"] for f in stats["file_stats"] if f["pct"] < 100.0)
+    if missing_counts:
+        import math
+        nm = len(missing_counts)
+        total_missing = sum(missing_counts)
+        m_max = missing_counts[-1]
+        use_log = m_max > 0 and missing_counts[0] > 0 and (m_max / missing_counts[0]) >= 100
+        if use_log:
+            log_min = math.log10(max(missing_counts[0], 1))
+            log_max = math.log10(m_max + 1)
+            log_step = (log_max - log_min) / bins
+            edges = [10 ** (log_min + i * log_step) for i in range(bins + 1)]
+            scale_note = "log scale"
+        else:
+            m_step = max((m_max + 0.1) / bins, 1)
+            edges = [i * m_step for i in range(bins + 1)]
+            scale_note = "linear scale"
+        m_buckets = [0] * bins
+        for v in missing_counts:
+            for bi in range(bins):
+                if edges[bi] <= v < edges[bi + 1] or (bi == bins - 1 and v == m_max):
+                    m_buckets[bi] += 1
+                    break
+            else:  # pragma: no cover
+                m_buckets[-1] += 1
+        m_labels = [f"{int(edges[i]):>5}–{int(edges[i + 1]):<5} lines" for i in range(bins)]
+        print(f"\n{'─' * width}")
+        print(f"  MISSING LINES DISTRIBUTION  (non-100% files: {nm})")
+        print(f"  [{scale_note}, {bins} bins]")
+        print()
+        ascii_histogram(m_buckets, m_labels, width=width)
+        print()
+        m_avg = total_missing / nm
+        print(
+            f"  p50: {int(percentile(missing_counts, 50)):<6} p75: {int(percentile(missing_counts, 75)):<6} "
+            f"p90: {int(percentile(missing_counts, 90)):<6} p99: {int(percentile(missing_counts, 99)):<6}"
+        )
+        print(f"  avg: {m_avg:<6.1f} max: {m_max:<6} total missing: {total_missing:,} lines")
+
+    if pragma_counts:
+        show_pragma_histogram(pragma_counts, bins=bins, blame_limit=blame_limit, width=width)
+
+    if show_blame:
+        # quality blame — cap ceiling at `threshold` so near-perfect codebases don't sweep 25% of files
+        q1 = percentile(cs, 25)
+        blame_ceiling = min(q1, threshold)
+        boundary_note = (
+            f"Q1={q1:.1f}% capped at {threshold:.0f}%" if q1 > threshold else f"Q1={q1:.1f}%"
+        )
+        blamed = sorted(
+            [
+                (f["pct"], f["path"], f["missing_count"], f["missing_lines"])
+                for f in stats["file_stats"]
+                if f["pct"] < blame_ceiling
+            ],
+            key=lambda x: x[0],
+        )
+        blame_header(f"below {blame_ceiling:.1f}% ({boundary_note})", len(blamed), blame_limit, width)
+        display = blamed if not blame_limit else blamed[:blame_limit]
+        if display:
+            for pct, path, missing_count, missing_lines in display:
+                print(f"  {pct:>5.1f}%  {missing_count:>4} lines missing  {path}")
+                ranges = format_line_ranges(missing_lines)
+                if ranges:
+                    print(f"         📍 {ranges}")
+        else:
+            print("  ✅  No files below quality boundary.")
+    print(f"\n{'═' * width}\n")