aiwaf 0.1.9.0.5__py3-none-any.whl → 0.1.9.0.7__py3-none-any.whl

aiwaf/__init__.py

@@ -1,6 +1,6 @@
 default_app_config = "aiwaf.apps.AiwafConfig"
 
-__version__ = "0.1.9.0.5"
+__version__ = "0.1.9.0.7"
 
 # Note: Middleware classes are available from aiwaf.middleware
 # Import them only when needed to avoid circular imports during Django app loading

aiwaf/blacklist_manager.py

@@ -12,7 +12,7 @@ class BlacklistManager:
             return  # Don't block exempted IPs
         
         store = get_blacklist_store()
-        store.add_ip(ip, reason)
+        store.block_ip(ip, reason)
 
     @staticmethod
     def is_blocked(ip):
@@ -29,9 +29,9 @@ class BlacklistManager:
     @staticmethod
     def all_blocked():
         store = get_blacklist_store()
-        return store.get_all()
+        return store.get_all_blocked_ips()
     
     @staticmethod
     def unblock(ip):
         store = get_blacklist_store()
-        store.remove_ip(ip)
+        store.unblock_ip(ip)

aiwaf/management/commands/add_exemption.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+
+from django.core.management.base import BaseCommand
+from aiwaf.storage import get_exemption_store
+
+class Command(BaseCommand):
+    help = 'Add IP to exemption list using Django models'
+
+    def add_arguments(self, parser):
+        parser.add_argument('ip', help='IP address to exempt')
+        parser.add_argument('--reason', default='Manual exemption', help='Reason for exemption')
+
+    def handle(self, *args, **options):
+        ip = options['ip']
+        reason = options['reason']
+        
+        self.stdout.write(f"Adding IP {ip} to exemption list...")
+        
+        exemption_store = get_exemption_store()
+        exemption_store.add_exemption(ip, reason)
+        
+        # Verify it was added
+        if exemption_store.is_exempted(ip):
+            self.stdout.write(self.style.SUCCESS(f"✅ Successfully exempted IP: {ip}"))
+        else:
+            self.stdout.write(self.style.ERROR(f"❌ Failed to exempt IP: {ip}"))
+        
+        # Show all exempted IPs
+        all_exempted = exemption_store.get_all_exempted_ips()
+        self.stdout.write(f"\nAll exempted IPs: {all_exempted}")

aiwaf/management/commands/add_ipexemption.py

@@ -17,7 +17,7 @@ class Command(BaseCommand):
         if store.is_exempted(ip):
             self.stdout.write(self.style.WARNING(f'IP {ip} is already exempted.'))
         else:
-            store.add_ip(ip, reason)
+            store.add_exemption(ip, reason)
             self.stdout.write(self.style.SUCCESS(f'IP {ip} added to exemption list.'))
             if reason:
                 self.stdout.write(self.style.SUCCESS(f'Reason: {reason}'))

aiwaf/management/commands/clear_cache.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+from django.core.management.base import BaseCommand
+from django.core.cache import cache
+
+class Command(BaseCommand):
+    help = 'Clear Django cache'
+
+    def handle(self, *args, **options):
+        cache.clear()
+        self.stdout.write(self.style.SUCCESS("✅ Django cache cleared successfully!"))
+        
+        # Also show what was cleared
+        self.stdout.write("🧹 Cleared all cached data including:")
+        self.stdout.write("   - Rate limiting data")
+        self.stdout.write("   - Blacklist cache")
+        self.stdout.write("   - AI anomaly data")
+        self.stdout.write("   - Honeypot timing data")

aiwaf/management/commands/debug_csv.py

@@ -101,7 +101,7 @@ class Command(BaseCommand):
             self.stdout.write(f"Before: IP {test_ip} exempted = {is_exempted_before}")
             
             # Add to exemption
-            exemption_store.add_ip(test_ip, "Test exemption from debug command")
+            exemption_store.add_exemption(test_ip, "Test exemption from debug command")
             self.stdout.write(f"✅ Added {test_ip} to exemption list")
             
             # Check if now exempted

aiwaf/management/commands/diagnose_blocking.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+
+from django.core.management.base import BaseCommand
+from django.core.cache import cache
+from aiwaf.blacklist_manager import BlacklistManager
+from aiwaf.storage import get_exemption_store, get_blacklist_store
+from aiwaf.utils import get_ip
+from django.test import RequestFactory
+
+class Command(BaseCommand):
+    help = 'Comprehensive diagnosis of blocking issues'
+
+    def add_arguments(self, parser):
+        parser.add_argument('--ip', default='97.187.30.95', help='IP address to test')
+        parser.add_argument('--clear-cache', action='store_true', help='Clear Django cache')
+
+    def handle(self, *args, **options):
+        test_ip = options['ip']
+        
+        self.stdout.write(f"\n🔍 Comprehensive Blocking Diagnosis for IP: {test_ip}")
+        self.stdout.write("=" * 60)
+        
+        if options['clear_cache']:
+            cache.clear()
+            self.stdout.write("🧹 Cleared Django cache")
+        
+        # 1. Check exemption status
+        exemption_store = get_exemption_store()
+        is_exempted = exemption_store.is_exempted(test_ip)
+        self.stdout.write(f"1. ✅ IP exempted in storage: {is_exempted}")
+        
+        # 2. Check blacklist status
+        blacklist_store = get_blacklist_store()
+        is_in_blacklist = blacklist_store.is_blocked(test_ip)
+        self.stdout.write(f"2. 🚫 IP in blacklist storage: {is_in_blacklist}")
+        
+        # 3. Check BlacklistManager final decision
+        manager_blocked = BlacklistManager.is_blocked(test_ip)
+        self.stdout.write(f"3. 🎯 BlacklistManager says blocked: {manager_blocked}")
+        
+        # 4. Check Django cache for blacklist entries
+        cache_key = f"blacklist:{test_ip}"
+        cached_value = cache.get(cache_key)
+        self.stdout.write(f"4. 💾 Cache value for blacklist:{test_ip}: {cached_value}")
+        
+        # 5. Test what IP would be detected from a request
+        factory = RequestFactory()
+        
+        # Test different scenarios
+        scenarios = [
+            ("Direct IP", {'REMOTE_ADDR': test_ip}),
+            ("X-Forwarded-For", {'HTTP_X_FORWARDED_FOR': test_ip}),
+            ("X-Real-IP", {'HTTP_X_REAL_IP': test_ip}),
+            ("CloudFlare", {'HTTP_CF_CONNECTING_IP': test_ip}),
+        ]
+        
+        self.stdout.write(f"\n5. 🌐 IP Detection Tests:")
+        for name, meta in scenarios:
+            request = factory.get('/', **meta)
+            detected_ip = get_ip(request)
+            self.stdout.write(f"   {name}: {detected_ip}")
+            if detected_ip == test_ip:
+                self.stdout.write(f"   ✅ Match!")
+            
+        # 6. Check rate limiting cache entries
+        self.stdout.write(f"\n6. 🚦 Rate Limiting Cache Entries:")
+        rate_keys = [
+            f"ratelimit:{test_ip}",
+            f"aiwaf:{test_ip}",
+            f"honeypot_get:{test_ip}"
+        ]
+        
+        for key in rate_keys:
+            value = cache.get(key)
+            if value:
+                self.stdout.write(f"   {key}: {value}")
+            else:
+                self.stdout.write(f"   {key}: None")
+        
+        # 7. Summary
+        self.stdout.write(f"\n📋 SUMMARY:")
+        if is_exempted and not manager_blocked:
+            self.stdout.write(self.style.SUCCESS("✅ IP should NOT be blocked"))
+            if options.get('still_blocked'):
+                self.stdout.write(self.style.WARNING("⚠️  If still blocked, check:"))
+                self.stdout.write("   - Web server logs (nginx, apache)")
+                self.stdout.write("   - Other middleware or security software")
+                self.stdout.write("   - Browser cache/cookies")
+        elif not is_exempted:
+            self.stdout.write(self.style.WARNING(f"⚠️  IP {test_ip} is NOT exempted"))
+        elif manager_blocked:
+            self.stdout.write(self.style.ERROR(f"❌ IP is being blocked despite exemption"))
+        
+        self.stdout.write(f"\n💡 To clear all caches and reset:")
+        self.stdout.write(f"   python manage.py shell -c \"from django.core.cache import cache; cache.clear()\"")
+        self.stdout.write(f"=" * 60)

aiwaf/management/commands/setup_models.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+
+from django.core.management.base import BaseCommand
+from django.db import transaction
+
+class Command(BaseCommand):
+    help = 'Create Django database migrations for AI-WAF models (after removing CSV support)'
+
+    def handle(self, *args, **options):
+        self.stdout.write("🔄 Creating AI-WAF database migrations...")
+        
+        try:
+            # Import the management command functions
+            from django.core.management import call_command
+            
+            # Create migrations for aiwaf app
+            self.stdout.write("Creating migrations for aiwaf models...")
+            call_command('makemigrations', 'aiwaf', verbosity=2)
+            
+            # Apply migrations
+            self.stdout.write("Applying migrations...")
+            call_command('migrate', 'aiwaf', verbosity=2)
+            
+            self.stdout.write(self.style.SUCCESS("✅ Successfully created and applied AI-WAF migrations!"))
+            self.stdout.write("")
+            self.stdout.write("🎯 Next steps:")
+            self.stdout.write("1. Add your IP to exemptions: python manage.py add_exemption YOUR_IP")
+            self.stdout.write("2. Test the system: python manage.py diagnose_blocking --ip YOUR_IP")
+            self.stdout.write("3. Clear any old cache: python manage.py clear_cache")
+            
+        except Exception as e:
+            self.stdout.write(self.style.ERROR(f"❌ Error during migration: {e}"))
+            self.stdout.write("You may need to run these commands manually:")
+            self.stdout.write("  python manage.py makemigrations aiwaf")
+            self.stdout.write("  python manage.py migrate aiwaf")

aiwaf/management/commands/test_exemption.py

@@ -63,7 +63,7 @@ class Command(BaseCommand):
             
             # Create the exemption
             try:
-                exemption_store.add_ip(test_ip, "Test exemption from debug")
+                exemption_store.add_exemption(test_ip, "Test exemption from debug")
                 self.stdout.write(self.style.SUCCESS("✅ Created test exemption"))
             except Exception as e:
                 self.stdout.write(self.style.ERROR(f"❌ Failed to create exemption: {e}"))

aiwaf/middleware_logger.py

@@ -1,44 +1,39 @@
 # aiwaf/middleware_logger.py
 
-import os
-import csv
 import time
 from datetime import datetime
 from django.conf import settings
 from django.utils.deprecation import MiddlewareMixin
+from django.utils import timezone
 from .utils import get_ip
 
+# Defer model imports to avoid AppRegistryNotReady during Django app loading
+RequestLog = None
+
+def _import_models():
+    """Import Django models only when needed and apps are ready."""
+    global RequestLog
+    
+    if RequestLog is not None:
+        return  # Already imported
+    
+    try:
+        from django.apps import apps
+        if apps.ready and apps.is_installed('aiwaf'):
+            from .models import RequestLog
+    except (ImportError, RuntimeError, Exception):
+        # Keep models as None if can't import
+        pass
+
 class AIWAFLoggerMiddleware(MiddlewareMixin):
     """
-    Middleware that logs requests to a CSV file for AI-WAF training.
+    Middleware that logs requests to Django models for AI-WAF training.
     Acts as a fallback when main access logs are unavailable.
     """
     
     def __init__(self, get_response):
         super().__init__(get_response)
-        self.log_file = getattr(settings, "AIWAF_MIDDLEWARE_LOG", "aiwaf_requests.log")
-        self.csv_format = getattr(settings, "AIWAF_MIDDLEWARE_CSV", True)
         self.log_enabled = getattr(settings, "AIWAF_MIDDLEWARE_LOGGING", False)
-        
-        # CSV file path (if using CSV format)
-        if self.csv_format and self.log_enabled:
-            self.csv_file = self.log_file.replace('.log', '.csv')
-            self._ensure_csv_header()
-    
-    def _ensure_csv_header(self):
-        """Ensure CSV file has proper header row"""
-        if not os.path.exists(self.csv_file):
-            # Create directory if it doesn't exist
-            csv_dir = os.path.dirname(self.csv_file)
-            if csv_dir and not os.path.exists(csv_dir):
-                os.makedirs(csv_dir, exist_ok=True)
-            
-            with open(self.csv_file, 'w', newline='', encoding='utf-8') as f:
-                writer = csv.writer(f)
-                writer.writerow([
-                    'timestamp', 'ip_address', 'method', 'path', 'status_code', 
-                    'response_time', 'user_agent', 'referer', 'content_length'
-                ])
     
     def process_request(self, request):
         """Store request start time"""
@@ -46,7 +41,7 @@ class AIWAFLoggerMiddleware(MiddlewareMixin):
         return None
     
     def process_response(self, request, response):
-        """Log the completed request"""
+        """Log the completed request to Django model"""
         if not self.log_enabled:
             return response
             
@@ -54,116 +49,81 @@ class AIWAFLoggerMiddleware(MiddlewareMixin):
         start_time = getattr(request, '_aiwaf_start_time', time.time())
         response_time = time.time() - start_time
         
-        # Extract request data
-        log_data = {
-            'timestamp': datetime.now().strftime('%d/%b/%Y:%H:%M:%S +0000'),
-            'ip_address': get_ip(request),
-            'method': request.method,
-            'path': request.path,
-            'status_code': response.status_code,
-            'response_time': f"{response_time:.3f}",
-            'user_agent': request.META.get('HTTP_USER_AGENT', '-'),
-            'referer': request.META.get('HTTP_REFERER', '-'),
-            'content_length': response.get('Content-Length', '-')
-        }
-        
-        if self.csv_format:
-            self._log_to_csv(log_data)
-        else:
-            self._log_to_text(log_data)
+        # Import models and log to database
+        _import_models()
+        if RequestLog is not None:
+            try:
+                RequestLog.objects.create(
+                    ip_address=get_ip(request),
+                    method=request.method,
+                    path=request.path[:500],  # Truncate long paths
+                    status_code=response.status_code,
+                    response_time=response_time,
+                    user_agent=request.META.get('HTTP_USER_AGENT', '')[:2000],  # Truncate long user agents
+                    referer=request.META.get('HTTP_REFERER', '')[:500],  # Truncate long referers
+                    content_length=response.get('Content-Length', '-'),
+                    timestamp=timezone.now()
+                )
+            except Exception as e:
+                # Fail silently to avoid breaking the application
+                pass
             
         return response
-    
-    def _log_to_csv(self, data):
-        """Write log entry to CSV file"""
-        try:
-            # Ensure directory exists before writing
-            csv_dir = os.path.dirname(self.csv_file)
-            if csv_dir and not os.path.exists(csv_dir):
-                os.makedirs(csv_dir, exist_ok=True)
-                
-            with open(self.csv_file, 'a', newline='', encoding='utf-8') as f:
-                writer = csv.writer(f)
-                writer.writerow([
-                    data['timestamp'], data['ip_address'], data['method'],
-                    data['path'], data['status_code'], data['response_time'],
-                    data['user_agent'], data['referer'], data['content_length']
-                ])
-        except Exception as e:
-            # Fail silently to avoid breaking the application
-            pass
-    
-    def _log_to_text(self, data):
-        """Write log entry in common log format"""
-        try:
-            # Common Log Format with response time
-            log_line = f'{data["ip_address"]} - - [{data["timestamp"]}] "{data["method"]} {data["path"]} HTTP/1.1" {data["status_code"]} {data["content_length"]} "{data["referer"]}" "{data["user_agent"]}" response-time={data["response_time"]}\n'
-            
-            with open(self.log_file, 'a', encoding='utf-8') as f:
-                f.write(log_line)
-        except Exception as e:
-            # Fail silently to avoid breaking the application
-            pass
 
 
-class AIWAFCSVLogParser:
+class AIWAFModelLogParser:
     """
-    Parser for AI-WAF CSV logs that converts them to the format expected by trainer.py
+    Parser for AI-WAF Django model logs that converts them to the format expected by trainer.py
     """
     
     @staticmethod
-    def parse_csv_log(csv_file_path):
+    def parse_model_logs():
         """
-        Parse CSV log file and return records in the format expected by trainer.py
+        Parse Django model logs and return records in the format expected by trainer.py
         Returns list of dictionaries with keys: ip, timestamp, path, status, referer, user_agent, response_time
         """
         records = []
         
-        if not os.path.exists(csv_file_path):
+        _import_models()
+        if RequestLog is None:
             return records
         
         try:
-            with open(csv_file_path, 'r', newline='', encoding='utf-8') as f:
-                reader = csv.DictReader(f)
-                for row in reader:
-                    try:
-                        # Convert timestamp to datetime object
-                        timestamp = datetime.strptime(row['timestamp'], '%d/%b/%Y:%H:%M:%S +0000')
-                        
-                        record = {
-                            'ip': row['ip_address'],
-                            'timestamp': timestamp,
-                            'path': row['path'],
-                            'status': row['status_code'],
-                            'referer': row['referer'],
-                            'user_agent': row['user_agent'],
-                            'response_time': float(row['response_time'])
-                        }
-                        records.append(record)
-                    except (ValueError, KeyError) as e:
-                        # Skip malformed rows
-                        continue
+            # Get all request logs
+            logs = RequestLog.objects.all().order_by('-timestamp')
+            
+            for log in logs:
+                record = {
+                    'ip': str(log.ip_address),
+                    'timestamp': log.timestamp,
+                    'path': log.path,
+                    'status': str(log.status_code),
+                    'referer': log.referer if log.referer else '-',
+                    'user_agent': log.user_agent if log.user_agent else '-',
+                    'response_time': log.response_time
+                }
+                records.append(record)
         except Exception as e:
-            # Return empty list if file can't be read
+            # Return empty list if models can't be accessed
             pass
             
         return records
     
     @staticmethod 
-    def get_log_lines_for_trainer(csv_file_path):
+    def get_log_lines_for_trainer():
         """
-        Convert CSV log to format compatible with trainer.py's _read_all_logs()
+        Convert Django model logs to format compatible with trainer.py's _read_all_logs()
         Returns list of log line strings
         """
-        records = AIWAFCSVLogParser.parse_csv_log(csv_file_path)
+        records = AIWAFModelLogParser.parse_model_logs()
         log_lines = []
         
         for record in records:
-            # Convert back to common log format that trainer.py expects
+            # Convert to common log format that trainer.py expects
             timestamp_str = record['timestamp'].strftime('%d/%b/%Y:%H:%M:%S +0000')
-            content_length = '-'  # We don't track this in our format
+            content_length = '-'  # We don't track this in detail
             
-            log_line = f'{record["ip"]} - - [{timestamp_str}] "GET {record["path"]} HTTP/1.1" {record["status"]} {content_length} "{record["referer"]}" "{record["user_agent"]}" response-time={record["response_time"]:.3f}'
+            log_line = f'{record["ip"]} - - [{timestamp_str}] "{record.get("method", "GET")} {record["path"]} HTTP/1.1" {record["status"]} {content_length} "{record["referer"]}" "{record["user_agent"]}" response-time={record["response_time"]:.3f}'
             log_lines.append(log_line)
             
         return log_lines

aiwaf/models.py

@@ -43,4 +43,31 @@ class IPExemption(models.Model):
     created_at = models.DateTimeField(auto_now_add=True)
 
     def __str__(self):
-        return f"{self.ip_address} (Exempted: {self.reason})"
+        return f"{self.ip_address} (Exempted: {self.reason})"
+
+
+# Model to store request logs for AI-WAF training
+class RequestLog(models.Model):
+    ip_address = models.GenericIPAddressField(db_index=True)
+    method = models.CharField(max_length=10)
+    path = models.CharField(max_length=500)
+    status_code = models.IntegerField()
+    response_time = models.FloatField()
+    user_agent = models.TextField(blank=True, default="")
+    referer = models.CharField(max_length=500, blank=True, default="")
+    content_length = models.CharField(max_length=20, blank=True, default="-")
+    timestamp = models.DateTimeField(auto_now_add=True)
+    
+    class Meta:
+        verbose_name = "Request Log"
+        verbose_name_plural = "Request Logs"
+        indexes = [
+            models.Index(fields=["ip_address"]),
+            models.Index(fields=["timestamp"]),
+            models.Index(fields=["status_code"]),
+            models.Index(fields=["method"]),
+        ]
+        ordering = ['-timestamp']
+    
+    def __str__(self):
+        return f"{self.ip_address} {self.method} {self.path} - {self.status_code}"