pintest-cli 0.2.0__py3-none-any.whl

pintest/range_set.py

@@ -0,0 +1,173 @@
+"""Efficient range-based storage for line numbers."""
+
+from typing import List, Tuple, Set
+
+
+class RangeSet:
+    """Efficient range-based storage with consistency maintenance."""
+    
+    def __init__(self, ranges: List[Tuple[int, int]] = None):
+        """Initialize with optional list of (start, end) ranges."""
+        self.ranges: List[Tuple[int, int]] = []
+        if ranges:
+            for start, end in ranges:
+                self.add_range(start, end)
+    
+    def add_lines(self, lines: Set[int]) -> None:
+        """Add individual line numbers, maintaining range consistency."""
+        for line in sorted(lines):
+            self.add_range(line, line)
+    
+    def add_range(self, start: int, end: int) -> None:
+        """Add a range, merging with overlapping/adjacent ranges."""
+        if start > end:
+            start, end = end, start
+        
+        # Find ranges that overlap or are adjacent to new range
+        merged_start = start
+        merged_end = end
+        new_ranges = []
+        merged = False
+        
+        for r_start, r_end in self.ranges:
+            # Check if ranges overlap or are adjacent (can be merged)
+            if r_end < start - 1:
+                # This range is completely before new range
+                new_ranges.append((r_start, r_end))
+            elif r_start > end + 1:
+                # This range is completely after new range
+                if not merged:
+                    new_ranges.append((merged_start, merged_end))
+                    merged = True
+                new_ranges.append((r_start, r_end))
+            else:
+                # Ranges overlap or are adjacent - merge them
+                merged_start = min(merged_start, r_start)
+                merged_end = max(merged_end, r_end)
+        
+        if not merged:
+            new_ranges.append((merged_start, merged_end))
+        
+        self.ranges = new_ranges
+    
+    def remove_lines(self, lines: Set[int]) -> None:
+        """Remove individual line numbers, splitting ranges if needed."""
+        for line in sorted(lines):
+            self.remove_range(line, line)
+    
+    def remove_range(self, start: int, end: int) -> None:
+        """Remove a range, splitting existing ranges if needed."""
+        if start > end:
+            start, end = end, start
+        
+        new_ranges = []
+        
+        for r_start, r_end in self.ranges:
+            if r_end < start or r_start > end:
+                # No overlap - keep range as is
+                new_ranges.append((r_start, r_end))
+            else:
+                # Overlap - split the range
+                if r_start < start:
+                    # Keep the part before removed range
+                    new_ranges.append((r_start, start - 1))
+                if r_end > end:
+                    # Keep the part after removed range
+                    new_ranges.append((end + 1, r_end))
+        
+        self.ranges = new_ranges
+    
+    def contains(self, line: int) -> bool:
+        """Check if a line number is covered by any range."""
+        # Binary search for efficiency
+        left, right = 0, len(self.ranges) - 1
+        
+        while left <= right:
+            mid = (left + right) // 2
+            start, end = self.ranges[mid]
+            
+            if line < start:
+                right = mid - 1
+            elif line > end:
+                left = mid + 1
+            else:
+                return True
+        
+        return False
+    
+    def intersects_any(self, line_numbers: Set[int]) -> bool:
+        """Check if any of the line numbers are covered."""
+        for line in line_numbers:
+            if self.contains(line):
+                return True
+        return False
+    
+    def intersection(self, other: 'RangeSet') -> 'RangeSet':
+        """Return intersection of two range sets."""
+        result = RangeSet()
+        
+        i = j = 0
+        while i < len(self.ranges) and j < len(other.ranges):
+            s1, e1 = self.ranges[i]
+            s2, e2 = other.ranges[j]
+            
+            # Find overlap
+            overlap_start = max(s1, s2)
+            overlap_end = min(e1, e2)
+            
+            if overlap_start <= overlap_end:
+                result.add_range(overlap_start, overlap_end)
+            
+            # Advance the range that ends first
+            if e1 < e2:
+                i += 1
+            else:
+                j += 1
+        
+        return result
+    
+    def union(self, other: 'RangeSet') -> 'RangeSet':
+        """Return union of two range sets."""
+        result = RangeSet(self.ranges[:])
+        for start, end in other.ranges:
+            result.add_range(start, end)
+        return result
+    
+    def to_list(self) -> List[Tuple[int, int]]:
+        """Export as list of (start, end) tuples."""
+        return self.ranges[:]
+    
+    def to_compact_string(self) -> str:
+        """Export as compact string representation: '1-5,10-12,20'."""
+        parts = []
+        for start, end in self.ranges:
+            if start == end:
+                parts.append(str(start))
+            else:
+                parts.append(f"{start}-{end}")
+        return ','.join(parts)
+    
+    @classmethod
+    def from_compact_string(cls, s: str) -> 'RangeSet':
+        """Import from compact string: '1-5,10-12,20'."""
+        if not s or not s.strip():
+            return cls()
+        
+        ranges = []
+        for part in s.split(','):
+            part = part.strip()
+            if '-' in part:
+                start, end = part.split('-', 1)
+                ranges.append((int(start), int(end)))
+            else:
+                val = int(part)
+                ranges.append((val, val))
+        
+        return cls(ranges)
+    
+    def __len__(self) -> int:
+        """Return total number of lines covered."""
+        return sum(end - start + 1 for start, end in self.ranges)
+    
+    def __repr__(self) -> str:
+        return f"RangeSet({self.ranges})"

pintest/test_mapping_db_v2.py

@@ -0,0 +1,381 @@
+"""Range-based test mapping storage for dramatic size reduction.
+
+Optimized version that stores line ranges instead of individual lines,
+reducing 12M entries to ~100K ranges (99%+ reduction).
+"""
+
+import sqlite3
+from pathlib import Path
+from typing import Set, List, Dict, Optional
+from dataclasses import dataclass
+from datetime import datetime
+
+from .range_set import RangeSet
+
+
+def normalize_test_name(test_name: str) -> str:
+    """
+    Normalize test name by removing parametrization.
+    
+    This aggregates parametrized tests into their base test name:
+    - test_foo[param1] -> test_foo
+    - test_foo[param2] -> test_foo
+    
+    This prevents:
+    1. Storing duplicate entries for each parameter combination
+    2. Issues with dynamic parameters that change between runs
+    
+    Args:
+        test_name: Full test name including parametrization
+        
+    Returns:
+        Base test name without parametrization
+    """
+    # Strip parametrization: test_name[params] -> test_name
+    if "[" in test_name and test_name.endswith("]"):
+        return test_name.split("[")[0]
+    return test_name
+
+
+
+@dataclass
+class TestMapping:
+    """Represents a test and the files/lines it covers."""
+    test_name: str
+    file_path: str
+    lines: RangeSet
+
+
+class TestMappingDBV2:
+    """SQLite database with range-based storage for test-to-code mappings."""
+    
+    SCHEMA_VERSION = 2
+    
+    def __init__(self, db_path: Path):
+        """
+        Initialize mapping database.
+        
+        Args:
+            db_path: Path to SQLite database file
+        """
+        self.db_path = Path(db_path)
+        self.conn: Optional[sqlite3.Connection] = None
+        
+    def __enter__(self):
+        """Context manager entry."""
+        self.connect()
+        return self
+        
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Context manager exit."""
+        self.close()
+        
+    def connect(self):
+        """Open database connection."""
+        self.conn = sqlite3.connect(str(self.db_path))
+        self.conn.row_factory = sqlite3.Row
+        
+        # Performance optimizations
+        self.conn.execute("PRAGMA journal_mode=WAL")
+        self.conn.execute("PRAGMA synchronous=NORMAL")
+        self.conn.execute("PRAGMA cache_size=-64000")  # 64MB cache
+        self.conn.execute("PRAGMA temp_store=MEMORY")
+        self.conn.execute("PRAGMA mmap_size=268435456")  # 256MB memory-mapped I/O
+        
+        # Auto-initialize schema if not present
+        if not self.is_initialized():
+            self.initialize_schema()
+        
+    def close(self):
+        """Close database connection."""
+        if self.conn:
+            self.conn.close()
+            self.conn = None
+    
+    def initialize_schema(self):
+        """Create database tables with range-based storage."""
+        cursor = self.conn.cursor()
+        
+        # Metadata table
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS metadata (
+                key TEXT PRIMARY KEY,
+                value TEXT NOT NULL
+            )
+        """)
+        
+        # Range-based test coverage table
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS test_coverage_ranges (
+                test_name TEXT NOT NULL,
+                file_path TEXT NOT NULL,
+                ranges TEXT NOT NULL,
+                PRIMARY KEY (test_name, file_path)
+            )
+        """)
+        
+        # Indexes for fast lookups
+        cursor.execute("""
+            CREATE INDEX IF NOT EXISTS idx_ranges_file_path 
+            ON test_coverage_ranges(file_path)
+        """)
+        
+        cursor.execute("""
+            CREATE INDEX IF NOT EXISTS idx_ranges_test_name 
+            ON test_coverage_ranges(test_name)
+        """)
+        
+        # Store schema version
+        cursor.execute("""
+            INSERT OR REPLACE INTO metadata (key, value)
+            VALUES ('schema_version', ?)
+        """, (str(self.SCHEMA_VERSION),))
+        
+        # Store creation timestamp
+        cursor.execute("""
+            INSERT OR IGNORE INTO metadata (key, value)
+            VALUES ('created_at', ?)
+        """, (datetime.utcnow().isoformat(),))
+        
+        self.conn.commit()
+    
+    def is_initialized(self) -> bool:
+        """Check if database schema is initialized."""
+        try:
+            cursor = self.conn.cursor()
+            cursor.execute("""
+                SELECT name FROM sqlite_master 
+                WHERE type='table' AND name='test_coverage_ranges'
+            """)
+            return cursor.fetchone() is not None
+        except sqlite3.Error:
+            return False
+    
+    def import_from_coverage(self, coverage_file: Path, incremental: bool = False):
+        """
+        Import test mappings from pytest .coverage file.
+        
+        Args:
+            coverage_file: Path to .coverage SQLite file
+            incremental: If True, merge with existing mappings
+        """
+        from .coverage_mapper import CoverageMapper
+        
+        mapper = CoverageMapper(coverage_file)
+        mapper.load_coverage()
+        
+        cursor = self.conn.cursor()
+        
+        # Build range-compressed mappings
+        test_file_ranges: Dict[Tuple[str, str], RangeSet] = {}
+        
+        for file_path, coverage_data in mapper.coverage_data.items():
+            for line_num, test_contexts in coverage_data.test_contexts.items():
+                for test_name in test_contexts:
+                    # Normalize test name to aggregate parametrized tests
+                    normalized_test = normalize_test_name(test_name)
+                    
+                    key = (normalized_test, file_path)
+                    if key not in test_file_ranges:
+                        test_file_ranges[key] = RangeSet()
+                    test_file_ranges[key].add_range(line_num, line_num)
+        
+        if incremental:
+            # Merge with existing ranges
+            for (test_name, file_path), new_ranges in test_file_ranges.items():
+                cursor.execute("""
+                    SELECT ranges FROM test_coverage_ranges
+                    WHERE test_name = ? AND file_path = ?
+                """, (test_name, file_path))
+                
+                row = cursor.fetchone()
+                if row:
+                    # Merge with existing
+                    existing = RangeSet.from_compact_string(row['ranges'])
+                    merged = existing.union(new_ranges)
+                    
+                    cursor.execute("""
+                        UPDATE test_coverage_ranges
+                        SET ranges = ?
+                        WHERE test_name = ? AND file_path = ?
+                    """, (merged.to_compact_string(), test_name, file_path))
+                else:
+                    # Insert new
+                    cursor.execute("""
+                        INSERT INTO test_coverage_ranges (test_name, file_path, ranges)
+                        VALUES (?, ?, ?)
+                    """, (test_name, file_path, new_ranges.to_compact_string()))
+        else:
+            # Replace all mappings
+            cursor.execute("DELETE FROM test_coverage_ranges")
+            
+            records = [
+                (test_name, file_path, ranges.to_compact_string())
+                for (test_name, file_path), ranges in test_file_ranges.items()
+            ]
+            
+            cursor.executemany("""
+                INSERT INTO test_coverage_ranges (test_name, file_path, ranges)
+                VALUES (?, ?, ?)
+            """, records)
+        
+        # Update metadata
+        cursor.execute("""
+            INSERT OR REPLACE INTO metadata (key, value)
+            VALUES ('last_import', ?)
+        """, (datetime.utcnow().isoformat(),))
+        
+        cursor.execute("""
+            INSERT OR REPLACE INTO metadata (key, value)
+            VALUES ('source_coverage_file', ?)
+        """, (str(coverage_file.absolute()),))
+        
+        cursor.execute("""
+            INSERT OR REPLACE INTO metadata (key, value)
+            VALUES ('total_tests', ?)
+        """, (str(len(set(k[0] for k in test_file_ranges.keys()))),))
+        
+        cursor.execute("""
+            INSERT OR REPLACE INTO metadata (key, value)
+            VALUES ('total_files', ?)
+        """, (str(len(mapper.coverage_data)),))
+        
+        self.conn.commit()
+        
+        return len(test_file_ranges)
+    
+    def find_tests_for_file_lines(
+        self, 
+        file_path: str, 
+        line_numbers: Set[int]
+    ) -> Set[str]:
+        """
+        Find all tests that cover specific lines in a file.
+        
+        Args:
+            file_path: Path to source file
+            line_numbers: Set of line numbers to check
+            
+        Returns:
+            Set of test names that cover those lines
+        """
+        if not line_numbers:
+            return set()
+        
+        cursor = self.conn.cursor()
+        file_path = str(file_path)
+        
+        # Query for all tests covering this file
+        cursor.execute("""
+            SELECT test_name, ranges
+            FROM test_coverage_ranges
+            WHERE file_path = ?
+        """, (file_path,))
+        
+        tests = set()
+        for row in cursor.fetchall():
+            ranges = RangeSet.from_compact_string(row['ranges'])
+            if ranges.intersects_any(line_numbers):
+                tests.add(row['test_name'])
+        
+        return tests
+    
+    def get_all_test_files(self, file_path: str) -> Set[str]:
+        """
+        Get all tests that cover any line in a file.
+        
+        Args:
+            file_path: Path to source file
+            
+        Returns:
+            Set of test names
+        """
+        cursor = self.conn.cursor()
+        file_path = str(file_path)
+        
+        cursor.execute("""
+            SELECT DISTINCT test_name
+            FROM test_coverage_ranges
+            WHERE file_path = ?
+        """, (file_path,))
+        
+        return {row['test_name'] for row in cursor.fetchall()}
+    
+    def get_metadata(self) -> Dict[str, str]:
+        """Get all metadata key-value pairs."""
+        cursor = self.conn.cursor()
+        cursor.execute("SELECT key, value FROM metadata")
+        return {row['key']: row['value'] for row in cursor.fetchall()}
+    
+    def get_stats(self) -> Dict[str, int]:
+        """Get database statistics."""
+        cursor = self.conn.cursor()
+        
+        cursor.execute("SELECT COUNT(DISTINCT test_name) FROM test_coverage_ranges")
+        total_tests = cursor.fetchone()[0]
+        
+        cursor.execute("SELECT COUNT(DISTINCT file_path) FROM test_coverage_ranges")
+        total_files = cursor.fetchone()[0]
+        
+        cursor.execute("SELECT COUNT(*) FROM test_coverage_ranges")
+        total_range_entries = cursor.fetchone()[0]
+        
+        # Calculate actual line coverage and compression ratio
+        cursor.execute("SELECT ranges FROM test_coverage_ranges")
+        total_lines = 0
+        total_ranges = 0
+        for row in cursor.fetchall():
+            rs = RangeSet.from_compact_string(row['ranges'])
+            total_lines += len(rs)
+            total_ranges += len(rs.ranges)
+        
+        return {
+            'total_tests': total_tests,
+            'total_files': total_files,
+            'total_range_entries': total_range_entries,
+            'total_lines_covered': total_lines,
+            'total_ranges': total_ranges,
+            'compression_ratio': round(total_lines / total_ranges, 2) if total_ranges > 0 else 0,
+            # Backward compatibility with old TestMappingDB
+            'total_mappings': total_lines,  # Equivalent to total line coverage
+        }
+    
+    def get_all_test_names(self) -> set:
+        """
+        Get all unique test names efficiently.
+        
+        Returns:
+            Set of all test names in the database
+        """
+        cursor = self.conn.cursor()
+        # This query benefits from the idx_test_name index
+        cursor.execute("SELECT DISTINCT test_name FROM test_coverage_ranges")
+        return {row[0] for row in cursor.fetchall()}
+    
+    def optimize(self):
+        """Optimize database for query performance."""
+        cursor = self.conn.cursor()
+        cursor.execute("ANALYZE")
+        self.conn.commit()
+    
+    def vacuum(self):
+        """Reclaim unused space."""
+        self.conn.execute("VACUUM")
+    
+    def is_stale(self, coverage_file: Path, max_age_hours: int = 24) -> bool:
+        """Check if mapping is stale compared to coverage file."""
+        if coverage_file.exists():
+            coverage_mtime = coverage_file.stat().st_mtime
+            if self.db_path.exists():
+                db_mtime = self.db_path.stat().st_mtime
+                if coverage_mtime > db_mtime:
+                    return True
+        
+        metadata = self.get_metadata()
+        if 'last_import' in metadata:
+            last_import = datetime.fromisoformat(metadata['last_import'])
+            age_hours = (datetime.utcnow() - last_import).total_seconds() / 3600
+            if age_hours > max_age_hours:
+                return True
+        
+        return False

pintest/update_mapping.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""
+Update test mapping database from coverage data.
+
+Usage:
+    python update_mapping.py --repo-root ~/workspace/myproject
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+# Add pintest to path
+SCRIPT_DIR = Path(__file__).parent
+sys.path.insert(0, str(SCRIPT_DIR))
+
+from pintest.test_mapping_db_v2 import TestMappingDBV2
+from pintest.pre_commit_hook import ensure_docker_containers, run_preflight_scripts
+
+
+def update_mapping(
+    repo_root: Path,
+    coverage_file: Path = None,
+    mapping_db: Path = None,
+    verbose: bool = False
+):
+    """
+    Update test mapping database from coverage file.
+    
+    Args:
+        repo_root: Repository root directory
+        coverage_file: Path to .coverage file
+        mapping_db: Path to mapping database file
+        verbose: Print verbose output
+    """
+    repo_root = repo_root.resolve()
+    
+    # Default paths
+    if coverage_file is None:
+        coverage_file = repo_root / ".coverage"
+    
+    if mapping_db is None:
+        mapping_db = repo_root / ".test_mapping.db"
+    
+    # Ensure Docker containers are running (if a compose file is present)
+    if verbose:
+        print("\n🔧 Pre-checks...")
+    
+    if not ensure_docker_containers(repo_root, verbose):
+        print("❌ Docker container check failed.", file=sys.stderr)
+        return 1
+    
+    if not run_preflight_scripts(repo_root, verbose):
+        print("❌ Preflight checks failed.", file=sys.stderr)
+        return 1
+    
+    if not coverage_file.exists():
+        print(f"❌ Error: Coverage file not found: {coverage_file}", file=sys.stderr)
+        print("\nGenerate coverage first:", file=sys.stderr)
+        print(f"  cd {repo_root}", file=sys.stderr)
+        print("  pytest --cov=src --cov-context=test --cov-report=", file=sys.stderr)
+        return 1
+    
+    print(f"📊 Updating test mapping database...")
+    print(f"   Coverage file: {coverage_file}")
+    print(f"   Mapping DB:    {mapping_db}")
+    
+    # Initialize database
+    with TestMappingDBV2(mapping_db) as db:
+        db.initialize_schema()
+        
+        # Import from coverage
+        print("\n📥 Importing coverage data...")
+        num_mappings = db.import_from_coverage(coverage_file)
+        
+        # Get stats
+        stats = db.get_stats()
+        metadata = db.get_metadata()
+        
+        print(f"\n✅ Test mapping database updated successfully!")
+        print(f"\n📈 Statistics:")
+        print(f"   Total tests:    {stats['total_tests']}")
+        print(f"   Total files:    {stats['total_files']}")
+        print(f"   Total mappings: {stats['total_mappings']}")
+        print(f"   Last import:    {metadata.get('last_import', 'N/A')}")
+        
+        if verbose:
+            print(f"\n🗄️  Database size: {mapping_db.stat().st_size / 1024:.1f} KB")
+    
+    return 0
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Update test mapping database from coverage data"
+    )
+    parser.add_argument(
+        "--repo-root",
+        type=Path,
+        default=Path.cwd(),
+        help="Repository root directory (default: current directory)"
+    )
+    parser.add_argument(
+        "--coverage-file",
+        type=Path,
+        help="Path to .coverage file (default: <repo>/.coverage)"
+    )
+    parser.add_argument(
+        "--mapping-db",
+        type=Path,
+        help="Path to mapping database (default: <repo>/.test_mapping.db)"
+    )
+    parser.add_argument(
+        "-v", "--verbose",
+        action="store_true",
+        help="Verbose output"
+    )
+    
+    args = parser.parse_args()
+    
+    return update_mapping(
+        args.repo_root,
+        args.coverage_file,
+        args.mapping_db,
+        args.verbose
+    )
+
+
+if __name__ == "__main__":
+    sys.exit(main())