aiwaf 0.1.8.7__py3-none-any.whl → 0.1.8.8__py3-none-any.whl

aiwaf/management/commands/aiwaf_logging.py

@@ -0,0 +1,166 @@
+from django.core.management.base import BaseCommand
+from django.conf import settings
+import os
+
+class Command(BaseCommand):
+    help = 'Manage AI-WAF middleware logging settings and view log status'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--enable',
+            action='store_true',
+            help='Enable middleware logging (shows settings to add)'
+        )
+        parser.add_argument(
+            '--disable',
+            action='store_true',
+            help='Disable middleware logging (shows settings to remove)'
+        )
+        parser.add_argument(
+            '--status',
+            action='store_true',
+            help='Show current middleware logging status'
+        )
+        parser.add_argument(
+            '--clear',
+            action='store_true',
+            help='Clear/delete middleware log files'
+        )
+
+    def handle(self, *args, **options):
+        if options['enable']:
+            self._show_enable_instructions()
+        elif options['disable']:
+            self._show_disable_instructions()
+        elif options['clear']:
+            self._clear_logs()
+        else:
+            self._show_status()
+
+    def _show_status(self):
+        """Show current middleware logging configuration"""
+        self.stdout.write(self.style.HTTP_INFO("🔍 AI-WAF Middleware Logging Status"))
+        self.stdout.write("")
+        
+        # Check settings
+        logging_enabled = getattr(settings, 'AIWAF_MIDDLEWARE_LOGGING', False)
+        log_file = getattr(settings, 'AIWAF_MIDDLEWARE_LOG', 'aiwaf_requests.log')
+        csv_format = getattr(settings, 'AIWAF_MIDDLEWARE_CSV', True)
+        csv_file = log_file.replace('.log', '.csv') if csv_format else None
+        
+        # Status
+        status_color = self.style.SUCCESS if logging_enabled else self.style.WARNING
+        self.stdout.write(f"Status: {status_color('ENABLED' if logging_enabled else 'DISABLED')}")
+        self.stdout.write(f"Log File: {log_file}")
+        if csv_format:
+            self.stdout.write(f"CSV File: {csv_file}")
+        self.stdout.write(f"Format: {'CSV' if csv_format else 'Text'}")
+        self.stdout.write("")
+        
+        # File existence and sizes
+        if logging_enabled:
+            self.stdout.write("📁 Log Files:")
+            
+            if csv_format and csv_file:
+                if os.path.exists(csv_file):
+                    size = os.path.getsize(csv_file)
+                    lines = self._count_csv_lines(csv_file)
+                    self.stdout.write(f"  ✅ {csv_file} ({size:,} bytes, {lines:,} entries)")
+                else:
+                    self.stdout.write(f"  ❌ {csv_file} (not found)")
+            
+            if os.path.exists(log_file):
+                size = os.path.getsize(log_file)
+                self.stdout.write(f"  ✅ {log_file} ({size:,} bytes)")
+            else:
+                self.stdout.write(f"  ❌ {log_file} (not found)")
+        
+        # Middleware check
+        middleware_list = getattr(settings, 'MIDDLEWARE', [])
+        middleware_installed = 'aiwaf.middleware_logger.AIWAFLoggerMiddleware' in middleware_list
+        
+        self.stdout.write("")
+        middleware_color = self.style.SUCCESS if middleware_installed else self.style.ERROR
+        self.stdout.write(f"Middleware: {middleware_color('INSTALLED' if middleware_installed else 'NOT INSTALLED')}")
+        
+        if logging_enabled and not middleware_installed:
+            self.stdout.write(self.style.WARNING("⚠️  Logging is enabled but middleware is not installed!"))
+
+    def _show_enable_instructions(self):
+        """Show instructions for enabling middleware logging"""
+        self.stdout.write(self.style.SUCCESS("🚀 Enable AI-WAF Middleware Logging"))
+        self.stdout.write("")
+        self.stdout.write("Add these settings to your Django settings.py:")
+        self.stdout.write("")
+        self.stdout.write(self.style.HTTP_INFO("# Enable AI-WAF middleware logging"))
+        self.stdout.write(self.style.HTTP_INFO("AIWAF_MIDDLEWARE_LOGGING = True"))
+        self.stdout.write(self.style.HTTP_INFO("AIWAF_MIDDLEWARE_LOG = 'aiwaf_requests.log'  # Optional"))
+        self.stdout.write(self.style.HTTP_INFO("AIWAF_MIDDLEWARE_CSV = True  # Optional (default: True)"))
+        self.stdout.write("")
+        self.stdout.write("Add middleware to MIDDLEWARE list (preferably near the end):")
+        self.stdout.write("")
+        self.stdout.write(self.style.HTTP_INFO("MIDDLEWARE = ["))
+        self.stdout.write(self.style.HTTP_INFO("    # ... your existing middleware ..."))
+        self.stdout.write(self.style.HTTP_INFO("    'aiwaf.middleware_logger.AIWAFLoggerMiddleware',"))
+        self.stdout.write(self.style.HTTP_INFO("]"))
+        self.stdout.write("")
+        self.stdout.write("Benefits:")
+        self.stdout.write("  ✅ Fallback when main access logs unavailable")
+        self.stdout.write("  ✅ CSV format for easy analysis") 
+        self.stdout.write("  ✅ Automatic integration with AI-WAF trainer")
+        self.stdout.write("  ✅ Captures response times for better detection")
+
+    def _show_disable_instructions(self):
+        """Show instructions for disabling middleware logging"""
+        self.stdout.write(self.style.WARNING("⏹️  Disable AI-WAF Middleware Logging"))
+        self.stdout.write("")
+        self.stdout.write("To disable, update your Django settings.py:")
+        self.stdout.write("")
+        self.stdout.write(self.style.HTTP_INFO("# Disable AI-WAF middleware logging"))
+        self.stdout.write(self.style.HTTP_INFO("AIWAF_MIDDLEWARE_LOGGING = False"))
+        self.stdout.write("")
+        self.stdout.write("And remove from MIDDLEWARE list:")
+        self.stdout.write("")
+        self.stdout.write(self.style.HTTP_INFO("MIDDLEWARE = ["))
+        self.stdout.write(self.style.HTTP_INFO("    # ... your existing middleware ..."))
+        self.stdout.write(self.style.HTTP_INFO("    # 'aiwaf.middleware_logger.AIWAFLoggerMiddleware',  # Remove this line"))
+        self.stdout.write(self.style.HTTP_INFO("]"))
+
+    def _clear_logs(self):
+        """Clear/delete middleware log files"""
+        log_file = getattr(settings, 'AIWAF_MIDDLEWARE_LOG', 'aiwaf_requests.log')
+        csv_format = getattr(settings, 'AIWAF_MIDDLEWARE_CSV', True)
+        csv_file = log_file.replace('.log', '.csv') if csv_format else None
+        
+        files_deleted = 0
+        
+        # Delete CSV file
+        if csv_file and os.path.exists(csv_file):
+            try:
+                os.remove(csv_file)
+                self.stdout.write(self.style.SUCCESS(f"✅ Deleted {csv_file}"))
+                files_deleted += 1
+            except Exception as e:
+                self.stdout.write(self.style.ERROR(f"❌ Failed to delete {csv_file}: {e}"))
+        
+        # Delete text log file
+        if os.path.exists(log_file):
+            try:
+                os.remove(log_file)
+                self.stdout.write(self.style.SUCCESS(f"✅ Deleted {log_file}"))
+                files_deleted += 1
+            except Exception as e:
+                self.stdout.write(self.style.ERROR(f"❌ Failed to delete {log_file}: {e}"))
+        
+        if files_deleted == 0:
+            self.stdout.write(self.style.WARNING("ℹ️  No log files found to delete"))
+        else:
+            self.stdout.write(self.style.SUCCESS(f"🗑️  Deleted {files_deleted} log file(s)"))
+
+    def _count_csv_lines(self, csv_file):
+        """Count lines in CSV file (excluding header)"""
+        try:
+            with open(csv_file, 'r', encoding='utf-8') as f:
+                return sum(1 for line in f) - 1  # Subtract header
+        except:
+            return 0

aiwaf/middleware_logger.py

@@ -0,0 +1,160 @@
+# aiwaf/middleware_logger.py
+
+import os
+import csv
+import time
+from datetime import datetime
+from django.conf import settings
+from django.utils.deprecation import MiddlewareMixin
+from .utils import get_ip
+
+class AIWAFLoggerMiddleware(MiddlewareMixin):
+    """
+    Middleware that logs requests to a CSV file for AI-WAF training.
+    Acts as a fallback when main access logs are unavailable.
+    """
+    
+    def __init__(self, get_response):
+        super().__init__(get_response)
+        self.log_file = getattr(settings, "AIWAF_MIDDLEWARE_LOG", "aiwaf_requests.log")
+        self.csv_format = getattr(settings, "AIWAF_MIDDLEWARE_CSV", True)
+        self.log_enabled = getattr(settings, "AIWAF_MIDDLEWARE_LOGGING", False)
+        
+        # CSV file path (if using CSV format)
+        if self.csv_format and self.log_enabled:
+            self.csv_file = self.log_file.replace('.log', '.csv')
+            self._ensure_csv_header()
+    
+    def _ensure_csv_header(self):
+        """Ensure CSV file has proper header row"""
+        if not os.path.exists(self.csv_file):
+            os.makedirs(os.path.dirname(self.csv_file), exist_ok=True) if os.path.dirname(self.csv_file) else None
+            with open(self.csv_file, 'w', newline='', encoding='utf-8') as f:
+                writer = csv.writer(f)
+                writer.writerow([
+                    'timestamp', 'ip_address', 'method', 'path', 'status_code', 
+                    'response_time', 'user_agent', 'referer', 'content_length'
+                ])
+    
+    def process_request(self, request):
+        """Store request start time"""
+        request._aiwaf_start_time = time.time()
+        return None
+    
+    def process_response(self, request, response):
+        """Log the completed request"""
+        if not self.log_enabled:
+            return response
+            
+        # Calculate response time
+        start_time = getattr(request, '_aiwaf_start_time', time.time())
+        response_time = time.time() - start_time
+        
+        # Extract request data
+        log_data = {
+            'timestamp': datetime.now().strftime('%d/%b/%Y:%H:%M:%S +0000'),
+            'ip_address': get_ip(request),
+            'method': request.method,
+            'path': request.path,
+            'status_code': response.status_code,
+            'response_time': f"{response_time:.3f}",
+            'user_agent': request.META.get('HTTP_USER_AGENT', '-'),
+            'referer': request.META.get('HTTP_REFERER', '-'),
+            'content_length': response.get('Content-Length', '-')
+        }
+        
+        if self.csv_format:
+            self._log_to_csv(log_data)
+        else:
+            self._log_to_text(log_data)
+            
+        return response
+    
+    def _log_to_csv(self, data):
+        """Write log entry to CSV file"""
+        try:
+            with open(self.csv_file, 'a', newline='', encoding='utf-8') as f:
+                writer = csv.writer(f)
+                writer.writerow([
+                    data['timestamp'], data['ip_address'], data['method'],
+                    data['path'], data['status_code'], data['response_time'],
+                    data['user_agent'], data['referer'], data['content_length']
+                ])
+        except Exception as e:
+            # Fail silently to avoid breaking the application
+            pass
+    
+    def _log_to_text(self, data):
+        """Write log entry in common log format"""
+        try:
+            # Common Log Format with response time
+            log_line = f'{data["ip_address"]} - - [{data["timestamp"]}] "{data["method"]} {data["path"]} HTTP/1.1" {data["status_code"]} {data["content_length"]} "{data["referer"]}" "{data["user_agent"]}" response-time={data["response_time"]}\n'
+            
+            with open(self.log_file, 'a', encoding='utf-8') as f:
+                f.write(log_line)
+        except Exception as e:
+            # Fail silently to avoid breaking the application
+            pass
+
+
+class AIWAFCSVLogParser:
+    """
+    Parser for AI-WAF CSV logs that converts them to the format expected by trainer.py
+    """
+    
+    @staticmethod
+    def parse_csv_log(csv_file_path):
+        """
+        Parse CSV log file and return records in the format expected by trainer.py
+        Returns list of dictionaries with keys: ip, timestamp, path, status, referer, user_agent, response_time
+        """
+        records = []
+        
+        if not os.path.exists(csv_file_path):
+            return records
+        
+        try:
+            with open(csv_file_path, 'r', newline='', encoding='utf-8') as f:
+                reader = csv.DictReader(f)
+                for row in reader:
+                    try:
+                        # Convert timestamp to datetime object
+                        timestamp = datetime.strptime(row['timestamp'], '%d/%b/%Y:%H:%M:%S +0000')
+                        
+                        record = {
+                            'ip': row['ip_address'],
+                            'timestamp': timestamp,
+                            'path': row['path'],
+                            'status': row['status_code'],
+                            'referer': row['referer'],
+                            'user_agent': row['user_agent'],
+                            'response_time': float(row['response_time'])
+                        }
+                        records.append(record)
+                    except (ValueError, KeyError) as e:
+                        # Skip malformed rows
+                        continue
+        except Exception as e:
+            # Return empty list if file can't be read
+            pass
+            
+        return records
+    
+    @staticmethod 
+    def get_log_lines_for_trainer(csv_file_path):
+        """
+        Convert CSV log to format compatible with trainer.py's _read_all_logs()
+        Returns list of log line strings
+        """
+        records = AIWAFCSVLogParser.parse_csv_log(csv_file_path)
+        log_lines = []
+        
+        for record in records:
+            # Convert back to common log format that trainer.py expects
+            timestamp_str = record['timestamp'].strftime('%d/%b/%Y:%H:%M:%S +0000')
+            content_length = '-'  # We don't track this in our format
+            
+            log_line = f'{record["ip"]} - - [{timestamp_str}] "GET {record["path"]} HTTP/1.1" {record["status"]} {content_length} "{record["referer"]}" "{record["user_agent"]}" response-time={record["response_time"]:.3f}'
+            log_lines.append(log_line)
+            
+        return log_lines

aiwaf/trainer.py

@@ -65,16 +65,31 @@ def remove_exempt_keywords() -> None:
 
 def _read_all_logs() -> list[str]:
     lines = []
+    
+    # First try to read from main access log
     if LOG_PATH and os.path.exists(LOG_PATH):
         with open(LOG_PATH, "r", errors="ignore") as f:
             lines.extend(f.readlines())
-    for p in sorted(glob.glob(f"{LOG_PATH}.*")):
-        opener = gzip.open if p.endswith(".gz") else open
-        try:
-            with opener(p, "rt", errors="ignore") as f:
-                lines.extend(f.readlines())
-        except OSError:
-            continue
+        for p in sorted(glob.glob(f"{LOG_PATH}.*")):
+            opener = gzip.open if p.endswith(".gz") else open
+            try:
+                with opener(p, "rt", errors="ignore") as f:
+                    lines.extend(f.readlines())
+            except OSError:
+                continue
+    
+    # If no lines found from main log, try AI-WAF middleware CSV log
+    if not lines:
+        middleware_csv = getattr(settings, "AIWAF_MIDDLEWARE_LOG", "aiwaf_requests.log").replace('.log', '.csv')
+        if os.path.exists(middleware_csv):
+            try:
+                from .middleware_logger import AIWAFCSVLogParser
+                csv_lines = AIWAFCSVLogParser.get_log_lines_for_trainer(middleware_csv)
+                lines.extend(csv_lines)
+                print(f"📋 Using AI-WAF middleware CSV log: {middleware_csv} ({len(csv_lines)} entries)")
+            except Exception as e:
+                print(f"⚠️  Failed to read middleware CSV log: {e}")
+    
     return lines
 
 

{aiwaf-0.1.8.7.dist-info → aiwaf-0.1.8.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aiwaf
-Version: 0.1.8.7
+Version: 0.1.8.8
 Summary: AI-powered Web Application Firewall
 Home-page: https://github.com/aayushgauba/aiwaf
 Author: Aayush Gauba
@@ -83,7 +83,28 @@ aiwaf/
   - Submit forms faster than `AIWAF_MIN_FORM_TIME` seconds (default: 1 second)
 
 - **UUID Tampering Protection**  
-  Blocks guessed or invalid UUIDs that don’t resolve to real models.
+  Blocks guessed or invalid UUIDs that don't resolve to real models.
+
+- **Built-in Request Logger**  
+  Optional middleware logger that captures requests to CSV:
+  - **Automatic fallback** when main access logs unavailable
+  - **CSV format** for easy analysis and training
+  - **Captures response times** for better anomaly detection
+  - **Zero configuration** - works out of the box
+
+
+**Exempt Path & IP Awareness**
+
+**Exempt Paths:**
+AI‑WAF automatically exempts common login paths (`/admin/`, `/login/`, `/accounts/login/`, etc.) from all blocking mechanisms. You can add additional exempt paths in your Django `settings.py`:
+
+```python
+AIWAF_EXEMPT_PATHS = [
+    "/api/webhooks/",
+    "/health/",
+    "/special-endpoint/",
+]
+```
 
 
 **Exempt Path & IP Awareness**
@@ -213,6 +234,42 @@ AIWAF_CSV_DATA_DIR = "aiwaf_data"  # Directory for CSV files
 
 ---
 
+### Built-in Request Logger (Optional)
+
+Enable AI-WAF's built-in request logger as a fallback when main access logs aren't available:
+
+```python
+# Enable middleware logging
+AIWAF_MIDDLEWARE_LOGGING = True                    # Enable/disable logging
+AIWAF_MIDDLEWARE_LOG = "aiwaf_requests.log"        # Log file path
+AIWAF_MIDDLEWARE_CSV = True                        # Use CSV format (recommended)
+```
+
+**Then add middleware to MIDDLEWARE list:**
+
+```python
+MIDDLEWARE = [
+    # ... your existing middleware ...
+    'aiwaf.middleware_logger.AIWAFLoggerMiddleware',  # Add near the end
+]
+```
+
+**Manage middleware logging:**
+
+```bash
+python manage.py aiwaf_logging --status    # Check logging status
+python manage.py aiwaf_logging --enable    # Show setup instructions  
+python manage.py aiwaf_logging --clear     # Clear log files
+```
+
+**Benefits:**
+- **Automatic fallback** when `AIWAF_ACCESS_LOG` unavailable
+- **CSV format** with precise timestamps and response times
+- **Zero configuration** - trainer automatically detects and uses CSV logs
+- **Lightweight** - fails silently to avoid breaking your application
+
+---
+
 ### Optional (defaults shown)
 
 ```python
@@ -244,14 +301,40 @@ Add in **this** order to your `MIDDLEWARE` list:
 ```python
 MIDDLEWARE = [
     "aiwaf.middleware.IPAndKeywordBlockMiddleware",
-    "aiwaf.middleware.RateLimitMiddleware",
+    "aiwaf.middleware.RateLimitMiddleware", 
     "aiwaf.middleware.AIAnomalyMiddleware",
     "aiwaf.middleware.HoneypotTimingMiddleware",
     "aiwaf.middleware.UUIDTamperMiddleware",
     # ... other middleware ...
+    "aiwaf.middleware_logger.AIWAFLoggerMiddleware",  # Optional: Add if using built-in logger
 ]
 ```
 
+> **⚠️ Order matters!** AI-WAF protection middleware should come early. The logger middleware should come near the end to capture final response data.
+
+---
+
+##  Running Detection & Training
+
+```bash
+python manage.py detect_and_train
+```
+
+### What happens:
+1. Read access logs (incl. rotated or gzipped) **OR** AI-WAF middleware CSV logs
+2. Auto‑block IPs with ≥ 6 total 404s
+3. Extract features & train IsolationForest
+4. Save `model.pkl`
+5. Extract top 10 dynamic keywords from 4xx/5xx
+6. Remove any keywords associated with newly exempt paths
+
+**Note:** If main access log (`AIWAF_ACCESS_LOG`) is unavailable, trainer automatically falls back to AI-WAF middleware CSV logs.
+
+---
+
+## 🧠 How It Works
+```
+
 ---
 
 ##  Running Detection & Training

{aiwaf-0.1.8.7.dist-info → aiwaf-0.1.8.8.dist-info}/RECORD

@@ -3,20 +3,22 @@ aiwaf/apps.py,sha256=nCez-Ptlv2kaEk5HenA8b1pATz1VfhrHP1344gwcY1A,142
 aiwaf/blacklist_manager.py,sha256=92ltIrFfv8WOC4CXwvNVZYfivkRZHGNg3E2QAbHQipQ,550
 aiwaf/decorators.py,sha256=IUKOdM_gdroffImRZep1g1wT6gNqD10zGwcp28hsJCs,825
 aiwaf/middleware.py,sha256=1JPrc0npI_a5bnB-thN0ME1ehfTbWBl1j9wTndZwRdQ,9505
+aiwaf/middleware_logger.py,sha256=uTYTvIc4Mv1pjY50aXaqQ5cWAO9qqquijAyVMs1KWlM,6517
 aiwaf/models.py,sha256=XaG1pd_oZu3y-fw66u4wblGlWcUY9gvsTNKGD0kQk7Y,1672
 aiwaf/storage.py,sha256=Z0KWArfLmOHnvUcL5aVx8W_aHMr-qoEW8FVGrM23BvA,11639
-aiwaf/trainer.py,sha256=qzQOtMW0_OA5JWWh6Znbc5BG3gcQ6Cb_NNWMOgLf3VQ,8487
+aiwaf/trainer.py,sha256=bgVoBewnNVMJdgxcNchfhsPOnFXxStoBOqNhFYnpsqs,9244
 aiwaf/utils.py,sha256=BJk5vJCYdGPl_4QQiknjhCbkzv5HZCXgFcBJDMJpHok,3390
 aiwaf/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aiwaf/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aiwaf/management/commands/add_ipexemption.py,sha256=srgdVPDJtF7G9GGIqaZ7L3qTuNheoS_uwlhlRO4W2bc,945
+aiwaf/management/commands/aiwaf_logging.py,sha256=FCIqULn2tii2vD9VxL7vk3PV4k4vr7kaA00KyaCExYY,7692
 aiwaf/management/commands/aiwaf_reset.py,sha256=0FIBqpZS8xgFFvAKJ-0zAC_-QNQwRkOHpXb8N-OdFr8,3740
 aiwaf/management/commands/detect_and_train.py,sha256=-o-LZ7QZ5GeJPCekryox1DGXKMmFEkwwrcDsiM166K0,269
 aiwaf/resources/model.pkl,sha256=5t6h9BX8yoh2xct85MXOO60jdlWyg1APskUOW0jZE1Y,1288265
 aiwaf/templatetags/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aiwaf/templatetags/aiwaf_tags.py,sha256=XXfb7Tl4DjU3Sc40GbqdaqOEtKTUKELBEk58u83wBNw,357
-aiwaf-0.1.8.7.dist-info/licenses/LICENSE,sha256=Ir8PX4dxgAcdB0wqNPIkw84fzIIRKE75NoUil9RX0QU,1069
-aiwaf-0.1.8.7.dist-info/METADATA,sha256=qAIF4-sV_Br3BPp6Ivn3fXzY9uWvdtMco1zXBlkKkW0,8664
-aiwaf-0.1.8.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-aiwaf-0.1.8.7.dist-info/top_level.txt,sha256=kU6EyjobT6UPCxuWpI_BvcHDG0I2tMgKaPlWzVxe2xI,6
-aiwaf-0.1.8.7.dist-info/RECORD,,
+aiwaf-0.1.8.8.dist-info/licenses/LICENSE,sha256=Ir8PX4dxgAcdB0wqNPIkw84fzIIRKE75NoUil9RX0QU,1069
+aiwaf-0.1.8.8.dist-info/METADATA,sha256=851Url25O97G0KGi3gHFF-zYSaI91BHiK7CbKJrLbk0,11261
+aiwaf-0.1.8.8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+aiwaf-0.1.8.8.dist-info/top_level.txt,sha256=kU6EyjobT6UPCxuWpI_BvcHDG0I2tMgKaPlWzVxe2xI,6
+aiwaf-0.1.8.8.dist-info/RECORD,,