setu-trafficmonitor 2.0.0__py3-none-any.whl

trafficmonitor/load_test_api.py

@@ -0,0 +1,307 @@
+"""
+API Load Testing Script - Tests the actual dashboard API endpoints
+Hammers the analytics API to find breaking points.
+
+Usage:
+    python request_logging/load_test_api.py --concurrent 50 --duration 30
+    python request_logging/load_test_api.py --preset extreme
+"""
+
+import argparse
+import concurrent.futures
+import json
+import random
+import sys
+import time
+from datetime import datetime, timedelta
+
+import requests
+from requests.auth import HTTPBasicAuth
+
+
+class APILoadTester:
+    """Tests the actual HTTP API endpoints."""
+
+    def __init__(self, base_url='http://localhost:8000', username=None, password=None):
+        self.base_url = base_url
+        self.session = requests.Session()
+        self.username = username
+        self.password = password
+        self.stats = {
+            'total': 0,
+            'success': 0,
+            'failed': 0,
+            'timeouts': 0,
+            'errors': {},
+            'response_times': [],
+        }
+
+    def setup_auth(self):
+        """Setup authentication."""
+        if self.username and self.password:
+            # Try to get session cookie first
+            login_url = f"{self.base_url}/admin/login/"
+            csrf_url = f"{self.base_url}/admin/"
+
+            # Get CSRF token
+            resp = self.session.get(csrf_url)
+            csrftoken = resp.cookies.get('csrftoken')
+
+            # Login
+            login_data = {
+                'username': self.username,
+                'password': self.password,
+                'csrfmiddlewaretoken': csrftoken,
+                'next': '/admin/',
+            }
+            self.session.post(login_url, data=login_data, headers={'Referer': login_url})
+
+            print(f"✓ Authenticated as {self.username}")
+        else:
+            print("⚠ Warning: No credentials provided, API calls may fail")
+
+    def test_endpoint(self, endpoint, params=None, timeout=10):
+        """Test a single endpoint."""
+        url = f"{self.base_url}{endpoint}"
+
+        start = time.time()
+        try:
+            response = self.session.get(url, params=params, timeout=timeout)
+            elapsed = time.time() - start
+
+            self.stats['total'] += 1
+            self.stats['response_times'].append(elapsed)
+
+            if response.status_code == 200:
+                self.stats['success'] += 1
+                return True, elapsed, response.status_code
+            else:
+                self.stats['failed'] += 1
+                error_key = f"HTTP_{response.status_code}"
+                self.stats['errors'][error_key] = self.stats['errors'].get(error_key, 0) + 1
+                return False, elapsed, response.status_code
+
+        except requests.Timeout:
+            elapsed = time.time() - start
+            self.stats['total'] += 1
+            self.stats['timeouts'] += 1
+            self.stats['errors']['TIMEOUT'] = self.stats['errors'].get('TIMEOUT', 0) + 1
+            return False, elapsed, 'TIMEOUT'
+
+        except Exception as e:
+            elapsed = time.time() - start
+            self.stats['total'] += 1
+            self.stats['failed'] += 1
+            error_key = type(e).__name__
+            self.stats['errors'][error_key] = self.stats['errors'].get(error_key, 0) + 1
+            return False, elapsed, str(e)
+
+    def generate_random_params(self):
+        """Generate random query parameters."""
+        ranges = ['today', 'yesterday', 'last_7_days', 'last_30_days']
+        methods = ['', 'GET', 'POST', 'PUT', 'DELETE']
+        statuses = ['', '200', '400', '404', '500']
+
+        params = {'range': random.choice(ranges)}
+
+        if random.random() > 0.5:
+            params['method'] = random.choice(methods)
+        if random.random() > 0.5:
+            params['status'] = random.choice(statuses)
+
+        return params
+
+    def run_concurrent_load(self, num_workers=10, duration=30, endpoints=None):
+        """Run concurrent load test."""
+        if endpoints is None:
+            endpoints = [
+                '/api/analytics/overview/',
+                '/api/analytics/chart/time-series/',
+                '/api/analytics/chart/status-codes/',
+                '/api/analytics/chart/methods/',
+                '/api/analytics/chart/endpoints/',
+                '/api/analytics/chart/performance/',
+                '/api/analytics/chart/heatmap/',
+                '/api/analytics/chart/errors/',
+            ]
+
+        print("=" * 70)
+        print("API LOAD TEST")
+        print("=" * 70)
+        print(f"Base URL:       {self.base_url}")
+        print(f"Workers:        {num_workers}")
+        print(f"Duration:       {duration}s")
+        print(f"Endpoints:      {len(endpoints)}")
+        print("=" * 70)
+
+        start_time = time.time()
+        end_time = start_time + duration
+
+        def worker():
+            """Worker function."""
+            requests_made = 0
+            while time.time() < end_time:
+                endpoint = random.choice(endpoints)
+                params = self.generate_random_params()
+
+                success, elapsed, status = self.test_endpoint(endpoint, params)
+
+                requests_made += 1
+
+                # Print progress
+                if requests_made % 10 == 0:
+                    print(f"Worker: {requests_made} requests, last: {status} in {elapsed*1000:.0f}ms")
+
+                # Small delay to avoid overwhelming
+                time.sleep(random.uniform(0.01, 0.1))
+
+        # Run workers
+        with concurrent.futures.ThreadPoolExecutor(max_workers=num_workers) as executor:
+            futures = [executor.submit(worker) for _ in range(num_workers)]
+
+            # Monitor progress
+            try:
+                while time.time() < end_time:
+                    time.sleep(5)
+                    elapsed = time.time() - start_time
+                    rps = self.stats['total'] / elapsed if elapsed > 0 else 0
+                    success_rate = (self.stats['success'] / self.stats['total'] * 100) if self.stats['total'] > 0 else 0
+
+                    print(f"\n[{elapsed:.0f}s] Requests: {self.stats['total']}, "
+                          f"RPS: {rps:.1f}, Success: {success_rate:.1f}%")
+
+            except KeyboardInterrupt:
+                print("\n\nStopping load test...")
+
+            # Wait for completion
+            concurrent.futures.wait(futures, timeout=10)
+
+        total_duration = time.time() - start_time
+        self._print_results(total_duration)
+
+    def _print_results(self, duration):
+        """Print test results."""
+        if not self.stats['response_times']:
+            print("\n⚠ No requests completed!")
+            return
+
+        sorted_times = sorted(self.stats['response_times'])
+        p50 = sorted_times[len(sorted_times) // 2]
+        p95 = sorted_times[int(len(sorted_times) * 0.95)] if len(sorted_times) > 20 else sorted_times[-1]
+        p99 = sorted_times[int(len(sorted_times) * 0.99)] if len(sorted_times) > 100 else sorted_times[-1]
+        avg = sum(sorted_times) / len(sorted_times)
+
+        print("\n" + "=" * 70)
+        print("API LOAD TEST RESULTS")
+        print("=" * 70)
+        print(f"Duration:            {duration:.2f}s")
+        print(f"Total Requests:      {self.stats['total']:,}")
+        print(f"Successful:          {self.stats['success']:,}")
+        print(f"Failed:              {self.stats['failed']:,}")
+        print(f"Timeouts:            {self.stats['timeouts']:,}")
+        print(f"Requests/sec:        {self.stats['total'] / duration:.2f}")
+        print(f"Success Rate:        {self.stats['success'] / self.stats['total'] * 100:.2f}%")
+
+        print("\nResponse Times:")
+        print(f"  Average:           {avg * 1000:.2f}ms")
+        print(f"  Median (p50):      {p50 * 1000:.2f}ms")
+        print(f"  95th percentile:   {p95 * 1000:.2f}ms")
+        print(f"  99th percentile:   {p99 * 1000:.2f}ms")
+        print(f"  Min:               {sorted_times[0] * 1000:.2f}ms")
+        print(f"  Max:               {sorted_times[-1] * 1000:.2f}ms")
+
+        if self.stats['errors']:
+            print("\nErrors:")
+            for error_type, count in sorted(self.stats['errors'].items(), key=lambda x: x[1], reverse=True):
+                print(f"  {error_type:.<30} {count:>6}")
+
+        print("=" * 70)
+
+        # Recommendations
+        self._print_recommendations(avg, p95, p99)
+
+    def _print_recommendations(self, avg, p95, p99):
+        """Print performance recommendations."""
+        print("\nPERFORMANCE ANALYSIS:")
+
+        if avg < 0.1:
+            print("  ✓ Excellent: Average response time < 100ms")
+        elif avg < 0.5:
+            print("  ✓ Good: Average response time < 500ms")
+        elif avg < 1.0:
+            print("  ⚠ Warning: Average response time approaching 1s")
+        else:
+            print("  ✗ Critical: Average response time > 1s - optimization needed!")
+
+        if p95 < 0.5:
+            print("  ✓ Excellent: 95th percentile < 500ms")
+        elif p95 < 1.0:
+            print("  ⚠ Warning: 95th percentile approaching 1s")
+        else:
+            print("  ✗ Critical: 95th percentile > 1s - some requests are very slow")
+
+        if p99 < 2.0:
+            print("  ✓ Good: 99th percentile < 2s")
+        else:
+            print("  ✗ Critical: 99th percentile > 2s - tail latency is high")
+
+        error_rate = self.stats['failed'] / self.stats['total'] * 100 if self.stats['total'] > 0 else 0
+        if error_rate == 0:
+            print("  ✓ Perfect: No errors!")
+        elif error_rate < 1:
+            print(f"  ✓ Good: Error rate < 1% ({error_rate:.2f}%)")
+        elif error_rate < 5:
+            print(f"  ⚠ Warning: Error rate {error_rate:.2f}%")
+        else:
+            print(f"  ✗ Critical: Error rate {error_rate:.2f}% - system unstable!")
+
+
+def main():
+    parser = argparse.ArgumentParser(description='API Load Testing')
+    parser.add_argument('--url', default='http://localhost:8000',
+                        help='Base URL (default: http://localhost:8000)')
+    parser.add_argument('--username', '-u', help='Username for authentication')
+    parser.add_argument('--password', '-p', help='Password for authentication')
+    parser.add_argument('--concurrent', '-c', type=int, default=10,
+                        help='Number of concurrent workers (default: 10)')
+    parser.add_argument('--duration', '-d', type=int, default=30,
+                        help='Test duration in seconds (default: 30)')
+    parser.add_argument('--preset', choices=['light', 'medium', 'heavy', 'extreme'],
+                        help='Use preset configuration')
+
+    args = parser.parse_args()
+
+    # Handle presets
+    if args.preset == 'light':
+        args.concurrent = 5
+        args.duration = 10
+    elif args.preset == 'medium':
+        args.concurrent = 20
+        args.duration = 30
+    elif args.preset == 'heavy':
+        args.concurrent = 50
+        args.duration = 60
+    elif args.preset == 'extreme':
+        args.concurrent = 100
+        args.duration = 120
+
+    # Create tester
+    tester = APILoadTester(
+        base_url=args.url,
+        username=args.username,
+        password=args.password
+    )
+
+    # Setup auth if credentials provided
+    if args.username and args.password:
+        tester.setup_auth()
+
+    # Run test
+    tester.run_concurrent_load(
+        num_workers=args.concurrent,
+        duration=args.duration
+    )
+
+
+if __name__ == '__main__':
+    main()

trafficmonitor/management/__init__.py

@@ -0,0 +1 @@
+# Management commands package

trafficmonitor/management/commands/__init__.py

@@ -0,0 +1 @@
+# Management commands

trafficmonitor/management/commands/cleanup_request_logs.py

@@ -0,0 +1,77 @@
+"""
+Management command to cleanup old request logs
+"""
+from django.core.management.base import BaseCommand
+from django.utils import timezone
+from datetime import timedelta
+from trafficmonitor.models import RequestLog
+from trafficmonitor.conf import TrafficMonitorConfig
+
+
+class Command(BaseCommand):
+    help = 'Cleanup old request logs based on retention policy'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--days',
+            type=int,
+            default=TrafficMonitorConfig.RETENTION_DAYS,
+            help=f'Number of days to retain (default: {TrafficMonitorConfig.RETENTION_DAYS})'
+        )
+        parser.add_argument(
+            '--batch-size',
+            type=int,
+            default=10000,
+            help='Batch size for deletion (default: 10000)'
+        )
+        parser.add_argument(
+            '--dry-run',
+            action='store_true',
+            help='Show what would be deleted without actually deleting'
+        )
+
+    def handle(self, *args, **options):
+        days = options['days']
+        batch_size = options['batch_size']
+        dry_run = options['dry_run']
+        
+        cutoff_date = timezone.now() - timedelta(days=days)
+        
+        # Count logs to be deleted
+        logs_to_delete = RequestLog.objects.filter(timestamp__lt=cutoff_date)
+        total_count = logs_to_delete.count()
+        
+        if total_count == 0:
+            self.stdout.write(
+                self.style.SUCCESS(f'No logs older than {days} days found.')
+            )
+            return
+        
+        if dry_run:
+            self.stdout.write(
+                self.style.WARNING(
+                    f'DRY RUN: Would delete {total_count} logs older than {cutoff_date}'
+                )
+            )
+            return
+        
+        # Delete in batches
+        deleted_total = 0
+        while True:
+            batch_ids = list(
+                logs_to_delete.values_list('id', flat=True)[:batch_size]
+            )
+            
+            if not batch_ids:
+                break
+            
+            deleted_count, _ = RequestLog.objects.filter(id__in=batch_ids).delete()
+            deleted_total += deleted_count
+            
+            self.stdout.write(f'Deleted {deleted_count} logs (total: {deleted_total}/{total_count})')
+        
+        self.stdout.write(
+            self.style.SUCCESS(
+                f'Successfully deleted {deleted_total} logs older than {days} days'
+            )
+        )