test-reporting 1.0.0__py3-none-any.whl

reporting/__init__.py

@@ -0,0 +1,6 @@
+"""
+Test Reporting System
+Automatic test result collection, analysis, and dashboard generation.
+"""
+
+__version__ = "1.0.0"

reporting/classifier.py

@@ -0,0 +1,109 @@
+"""
+Failure Classification
+Automatically categorizes test failures into different types.
+"""
+import re
+from typing import Optional, Dict, Any
+from .config import ReportingConfig
+
+
+class FailureClassifier:
+    """Classifies test failures into categories."""
+    
+    FAILURE_TYPES = {
+        'ASSERTION': 'Hard Failure - Assertion Error',
+        'UI_TIMEOUT': 'Soft Failure - UI Timeout',
+        'API_TIMEOUT': 'Soft Failure - API Timeout',
+        'ENVIRONMENT': 'Environment Issue',
+        'FLAKY': 'Flaky Test - Passed on Retry',
+        'UNKNOWN': 'Unknown Error',
+    }
+    
+    @staticmethod
+    def classify(
+        exception_type: str,
+        exception_message: str,
+        test_name: str = '',
+        duration_ms: float = 0,
+        retry_count: int = 0
+    ) -> Dict[str, Any]:
+        """
+        Classify a test failure.
+        
+        Args:
+            exception_type: Type of exception (e.g., 'AssertionError')
+            exception_message: Exception message
+            test_name: Name of the test
+            duration_ms: Test duration in milliseconds
+            retry_count: Number of retries (0 if passed first time)
+            
+        Returns:
+            Dict with classification info
+        """
+        failure_type = 'UNKNOWN'
+        is_soft_fail = False
+        category = 'functional'
+        
+        # Check if passed on retry (flaky)
+        if retry_count > 0:
+            failure_type = 'FLAKY'
+            is_soft_fail = True
+            category = 'stability'
+        
+        # Check for assertion errors (hard failures)
+        elif 'AssertionError' in exception_type or 'assert' in exception_message.lower():
+            failure_type = 'ASSERTION'
+            is_soft_fail = False
+            category = 'functional'
+        
+        # Check for UI timeouts (soft failures)
+        elif any(pattern in exception_message for pattern in ReportingConfig.SOFT_FAIL_PATTERNS):
+            failure_type = 'UI_TIMEOUT'
+            is_soft_fail = True
+            category = 'performance'
+        
+        # Check for environment issues
+        elif any(pattern in exception_message.lower() for pattern in ReportingConfig.ENVIRONMENT_FAIL_PATTERNS):
+            failure_type = 'ENVIRONMENT'
+            is_soft_fail = True
+            category = 'infrastructure'
+        
+        # Check for API timeouts
+        elif 'timeout' in exception_message.lower() and duration_ms > ReportingConfig.TIMEOUT_API_MAX:
+            failure_type = 'API_TIMEOUT'
+            is_soft_fail = True
+            category = 'performance'
+        
+        return {
+            'failure_type': failure_type,
+            'failure_description': FailureClassifier.FAILURE_TYPES.get(failure_type, 'Unknown'),
+            'is_soft_fail': is_soft_fail,
+            'category': category,
+            'should_count_in_pass_rate': not is_soft_fail,
+        }
+    
+    @staticmethod
+    def extract_timeout_value(error_message: str) -> Optional[int]:
+        """Extract timeout value from error message."""
+        # Look for patterns like "Timeout 30000ms exceeded"
+        match = re.search(r'Timeout (\d+)ms', error_message)
+        if match:
+            return int(match.group(1))
+        
+        # Look for patterns like "timeout of 30000ms exceeded"
+        match = re.search(r'timeout of (\d+)ms', error_message)
+        if match:
+            return int(match.group(1))
+        
+        return None
+    
+    @staticmethod
+    def is_performance_issue(duration_ms: float, failure_type: str) -> bool:
+        """Check if failure is performance-related."""
+        if failure_type in ['UI_TIMEOUT', 'API_TIMEOUT']:
+            return True
+        
+        if duration_ms > ReportingConfig.TIMEOUT_UI_WARNING:
+            return True
+        
+        return False

reporting/cli.py

@@ -0,0 +1,209 @@
+"""
+CLI tool for reporting system
+"""
+import sys
+import webbrowser
+from pathlib import Path
+from .dashboard_generator_v2 import EnhancedDashboardGenerator
+from .overview_dashboard import OverviewDashboardGenerator
+from .history_dashboard import HistoryDashboardGenerator
+from .projects_dashboard import ProjectsDashboardGenerator
+from .config import ReportingConfig
+
+
+def generate_dashboard():
+    """Generate all dashboards (projects landing, overview, run-details, and history)."""
+    print("[Reporting] Generating dashboards...")
+    
+    # Create config instance
+    config = ReportingConfig()
+    print(f"[Reporting] Project: {config.PROJECT_NAME}")
+    print(f"[Reporting] Database: {config.DB_PATH}")
+    
+    # Generate projects landing page (index.html)
+    print("\n[Reporting] Generating projects landing page...")
+    projects_gen = ProjectsDashboardGenerator(config=config)
+    projects_path = projects_gen.generate()
+    
+    # Generate overview dashboard (overview.html)
+    print("\n[Reporting] Generating overview dashboard...")
+    overview_gen = OverviewDashboardGenerator(config=config)
+    overview_path = overview_gen.generate(output_path=config.DASHBOARD_DIR / 'overview.html')
+    
+    # Generate run-details dashboard
+    print("\n[Reporting] Generating run-details dashboard...")
+    details_gen = EnhancedDashboardGenerator(config=config)
+    details_path = details_gen.generate()
+    
+    # Generate history dashboard
+    print("\n[Reporting] Generating test history dashboard...")
+    history_gen = HistoryDashboardGenerator(config=config)
+    history_path = history_gen.generate()
+    
+    print(f"\n[Reporting] All dashboards generated successfully!")
+    print(f"  Projects: {projects_path}")
+    print(f"  Overview: {overview_path}")
+    print(f"  Details:  {details_path}")
+    print(f"  History:  {history_path}")
+    
+    return projects_path
+
+
+def open_dashboard():
+    """Generate and open overview dashboard in browser."""
+    overview_path = generate_dashboard()
+    print(f"\n[Reporting] Opening overview dashboard in browser...")
+    webbrowser.open(f"file:///{overview_path.absolute()}")
+
+
+def show_stats():
+    """Show quick stats from latest run."""
+    import json
+    
+    if not ReportingConfig.LATEST_JSON_PATH.exists():
+        print("[ERROR] No test data found. Run tests first.")
+        return
+    
+    with open(ReportingConfig.LATEST_JSON_PATH, 'r') as f:
+        data = json.load(f)
+    
+    metrics = data.get('metrics', {})
+    status = data.get('status', {})
+    
+    print("\n" + "="*60)
+    print(f"  TEST RESULTS SUMMARY")
+    print("="*60)
+    print(f"Build: #{data.get('build_number', 'N/A')}")
+    print(f"Suite: {data.get('suite_name', 'N/A')}")
+    print(f"Time:  {data.get('timestamp', 'N/A')}")
+    print("-"*60)
+    print(f"Health Score:         {metrics.get('health_score', 0)}/100")
+    print(f"Pass Rate:            {metrics.get('pass_rate', 0):.1f}%")
+    print(f"Functional Pass Rate: {metrics.get('functional_pass_rate', 0):.1f}%")
+    print(f"Total Tests:          {metrics.get('total_tests', 0)}")
+    print(f"  Passed:             {metrics.get('passed', 0)}")
+    print(f"  Failed:             {metrics.get('failed', 0)}")
+    print(f"  Skipped:            {metrics.get('skipped', 0)}")
+    print("="*60)
+    
+    if data.get('top_failures'):
+        print("\nTop Failures:")
+        for i, failure in enumerate(data['top_failures'][:3], 1):
+            print(f"\n{i}. {failure['test_name']}")
+            print(f"   File: {failure['file_name']}")
+            print(f"   Type: {failure.get('failure_type', 'UNKNOWN')}")
+
+
+def show_suites():
+    """Show statistics for all test suites."""
+    from .storage import TestResultStorage
+    
+    storage = TestResultStorage()
+    suites = storage.get_suite_statistics()
+    
+    if not suites:
+        print("[ERROR] No suite data found. Run test suites first.")
+        return
+    
+    print("\n" + "="*80)
+    print(f"  TEST SUITE STATISTICS")
+    print("="*80)
+    
+    for suite in suites:
+        suite_name = suite['suite_name']
+        total_runs = suite['total_runs']
+        avg_duration = suite['avg_duration']
+        
+        # Format duration
+        if avg_duration >= 60:
+            mins = int(avg_duration // 60)
+            secs = int(avg_duration % 60)
+            duration_str = f"{mins}m {secs}s"
+        else:
+            duration_str = f"{avg_duration:.1f}s"
+        
+        print(f"\nSuite: {suite_name}")
+        print(f"  Total Runs:     {total_runs}")
+        print(f"  Avg Pass Rate:  {suite['avg_pass_rate']:.1f}%")
+        print(f"  Avg Duration:   {duration_str}")
+        print(f"  Total Tests:    {suite['total_tests']}")
+        print(f"  Last Run:       {suite['last_run']}")
+    
+    print("\n" + "="*80)
+
+
+def cleanup_old_data():
+    """Clean up old test data based on retention policy."""
+    from .storage import TestResultStorage
+    
+    config = ReportingConfig()
+    storage = TestResultStorage(config=config)
+    
+    print(f"\n[Reporting] Cleaning up data older than {config.RETENTION_DAYS} days...")
+    print(f"[Reporting] Database: {config.DB_PATH}")
+    
+    # Get count before cleanup
+    import sqlite3
+    with sqlite3.connect(config.DB_PATH) as conn:
+        cursor = conn.execute("SELECT COUNT(*) FROM test_runs")
+        before_count = cursor.fetchone()[0]
+        
+        cursor = conn.execute(f"SELECT COUNT(*) FROM test_runs WHERE timestamp < datetime('now', '-{config.RETENTION_DAYS} days')")
+        to_delete = cursor.fetchone()[0]
+    
+    if to_delete == 0:
+        print(f"[Reporting] No data older than {config.RETENTION_DAYS} days found. Nothing to clean up.")
+        return
+    
+    print(f"[Reporting] Found {to_delete} test runs to delete (out of {before_count} total)")
+    
+    # Confirm deletion
+    response = input(f"[Reporting] Delete {to_delete} old test runs? (yes/no): ")
+    if response.lower() not in ['yes', 'y']:
+        print("[Reporting] Cleanup cancelled.")
+        return
+    
+    # Perform cleanup
+    storage.cleanup_old_data()
+    
+    # Get count after cleanup
+    with sqlite3.connect(config.DB_PATH) as conn:
+        cursor = conn.execute("SELECT COUNT(*) FROM test_runs")
+        after_count = cursor.fetchone()[0]
+    
+    deleted = before_count - after_count
+    print(f"[Reporting] ✅ Cleanup complete! Deleted {deleted} test runs.")
+    print(f"[Reporting] Remaining: {after_count} test runs")
+
+
+def main():
+    """Main CLI entry point."""
+    if len(sys.argv) < 2:
+        print("Usage: python -m reporting.cli <command>")
+        print("\nCommands:")
+        print("  generate  - Generate dashboard HTML")
+        print("  open      - Generate and open dashboard in browser")
+        print("  stats     - Show quick stats from latest run")
+        print("  suites    - Show statistics for all test suites")
+        print("  cleanup   - Clean up old test data (based on retention_days)")
+        return
+    
+    command = sys.argv[1]
+    
+    if command == 'generate':
+        generate_dashboard()
+    elif command == 'open':
+        open_dashboard()
+    elif command == 'stats':
+        show_stats()
+    elif command == 'suites':
+        show_suites()
+    elif command == 'cleanup':
+        cleanup_old_data()
+    else:
+        print(f"Unknown command: {command}")
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

reporting/config.py

@@ -0,0 +1,114 @@
+"""
+Reporting Configuration
+"""
+import os
+import configparser
+from pathlib import Path
+
+
+class ReportingConfig:
+    """Configuration for the reporting system."""
+    
+    def __init__(self, config_file='reporting.ini'):
+        """Initialize config from file or environment variables."""
+        self.config = configparser.ConfigParser()
+        
+        # Try to read config file from current directory
+        config_path = Path.cwd() / config_file
+        if config_path.exists():
+            self.config.read(config_path)
+            self._load_from_file()
+        else:
+            self._load_from_defaults()
+    
+    def _load_from_file(self):
+        """Load configuration from ini file."""
+        # Project info
+        self.PROJECT_NAME = self.config.get('reporting', 'project_name', 
+                                           fallback='test-automation-project')
+        
+        # Paths - support shared database
+        db_path_str = self.config.get('reporting', 'db_path', 
+                                     fallback='test_results.db')
+        self.DB_PATH = Path(db_path_str)
+        
+        dashboard_dir_str = self.config.get('reporting', 'dashboard_dir', 
+                                           fallback='dashboard')
+        self.DASHBOARD_DIR = Path(dashboard_dir_str)
+        
+        # Other directories (project-specific)
+        base_dir = Path.cwd()
+        self.BASE_DIR = base_dir
+        self.REPORTS_DIR = base_dir / 'reports'
+        self.SCREENSHOTS_DIR = base_dir / 'screenshots'
+        self.TRACES_DIR = base_dir / 'traces'
+        self.LATEST_JSON_PATH = self.REPORTS_DIR / 'latest.json'
+        
+        # Retention
+        self.RETENTION_DAYS = self.config.getint('reporting', 'retention_days', 
+                                                 fallback=365)
+    
+    def _load_from_defaults(self):
+        """Load default configuration."""
+        # Project info
+        self.PROJECT_NAME = os.getenv('PROJECT_NAME', 'test-automation-project')
+        
+        # Paths
+        self.BASE_DIR = Path(__file__).parent.parent
+        self.REPORTS_DIR = self.BASE_DIR / 'reports'
+        self.DASHBOARD_DIR = self.BASE_DIR / 'dashboard'
+        self.SCREENSHOTS_DIR = self.BASE_DIR / 'screenshots'
+        self.TRACES_DIR = self.BASE_DIR / 'traces'
+        
+        # Database - default to local location
+        self.DB_PATH = Path('test_results.db')
+        self.LATEST_JSON_PATH = self.REPORTS_DIR / 'latest.json'
+        
+        # Retention
+        self.RETENTION_DAYS = 365
+    
+    # Timeout thresholds (milliseconds)
+    TIMEOUT_UI_WARNING = 30000  # Warn if UI action > 30s
+    TIMEOUT_UI_MAX = 60000      # Hard fail if > 60s
+    TIMEOUT_API_WARNING = 10000
+    TIMEOUT_API_MAX = 30000
+    
+    # Failure classification
+    SOFT_FAIL_PATTERNS = [
+        'TimeoutError',
+        'timeout',
+        'wait_for_selector',
+        'wait_for_load_state',
+        'Page.waitForSelector',
+    ]
+    
+    ENVIRONMENT_FAIL_PATTERNS = [
+        'connection',
+        'network',
+        'database',
+        'ECONNREFUSED',
+        'ETIMEDOUT',
+    ]
+    
+    # Data retention
+    RETENTION_MAX_RUNS = 1000
+    
+    # Dashboard settings
+    DASHBOARD_AUTO_REFRESH_SECONDS = 30
+    DASHBOARD_SHOW_SCREENSHOTS = True
+    DASHBOARD_SHOW_TRACES = True
+    
+    @property
+    def DASHBOARD_TITLE(self):
+        """Generate dashboard title with project name."""
+        return f"{self.PROJECT_NAME} - Test Automation Dashboard"
+    
+    def ensure_directories(self):
+        """Create necessary directories if they don't exist."""
+        self.REPORTS_DIR.mkdir(exist_ok=True, parents=True)
+        self.DASHBOARD_DIR.mkdir(exist_ok=True, parents=True)
+        self.SCREENSHOTS_DIR.mkdir(exist_ok=True, parents=True)
+        self.TRACES_DIR.mkdir(exist_ok=True, parents=True)
+        
+        # Ensure shared database directory exists
+        self.DB_PATH.parent.mkdir(exist_ok=True, parents=True)