aiwaf 0.1.7.6__py3-none-any.whl → 0.1.7.8__py3-none-any.whl

aiwaf/management/commands/add_ipexemption.py

@@ -0,0 +1,22 @@
+from django.core.management.base import BaseCommand, CommandError
+from aiwaf.models import IPExemption
+
+class Command(BaseCommand):
+    help = 'Add an IP address to the IPExemption list (prevents blacklisting)'
+
+    def add_arguments(self, parser):
+        parser.add_argument('ip', type=str, help='IP address to exempt')
+        parser.add_argument('--reason', type=str, default='', help='Reason for exemption (optional)')
+
+    def handle(self, *args, **options):
+        ip = options['ip']
+        reason = options['reason']
+        obj, created = IPExemption.objects.get_or_create(ip_address=ip, defaults={'reason': reason})
+        if not created:
+            self.stdout.write(self.style.WARNING(f'IP {ip} is already exempted.'))
+        else:
+            self.stdout.write(self.style.SUCCESS(f'IP {ip} added to exemption list.'))
+        if reason:
+            obj.reason = reason
+            obj.save()
+            self.stdout.write(self.style.SUCCESS(f'Reason set to: {reason}'))

aiwaf/middleware.py

@@ -16,7 +16,9 @@ from django.apps import apps
 from django.urls import get_resolver
 from .trainer import STATIC_KW, STATUS_IDX, is_exempt_path, path_exists_in_django
 from .blacklist_manager import BlacklistManager
-from .models import DynamicKeyword
+from .models import DynamicKeyword, IPExemption
+def is_ip_exempted(ip):
+    return IPExemption.objects.filter(ip_address=ip).exists()
 
 def is_exempt_path(path):
     path = path.lower()
@@ -77,6 +79,8 @@ class IPAndKeywordBlockMiddleware:
             return self.get_response(request)
         ip = get_ip(request)
         path = raw_path.lstrip("/")
+        if is_ip_exempted(ip):
+            return self.get_response(request)
         if BlacklistManager.is_blocked(ip):
             return JsonResponse({"error": "blocked"}, status=403)
         segments = [seg for seg in re.split(r"\W+", path) if len(seg) > 3]
@@ -95,8 +99,9 @@ class IPAndKeywordBlockMiddleware:
         }
         for seg in segments:
             if seg in suspicious_kw:
-                BlacklistManager.block(ip, f"Keyword block: {seg}")
-                return JsonResponse({"error": "blocked"}, status=403)
+                if not is_ip_exempted(ip):
+                    BlacklistManager.block(ip, f"Keyword block: {seg}")
+                    return JsonResponse({"error": "blocked"}, status=403)
         return self.get_response(request)
 
 
@@ -120,8 +125,9 @@ class RateLimitMiddleware:
         timestamps.append(now)
         cache.set(key, timestamps, timeout=self.WINDOW)
         if len(timestamps) > self.FLOOD:
-            BlacklistManager.block(ip, "Flood pattern")
-            return JsonResponse({"error": "blocked"}, status=403)
+            if not is_ip_exempted(ip):
+                BlacklistManager.block(ip, "Flood pattern")
+                return JsonResponse({"error": "blocked"}, status=403)
         if len(timestamps) > self.MAX:
             return JsonResponse({"error": "too_many_requests"}, status=429)
         return self.get_response(request)
@@ -141,6 +147,8 @@ class AIAnomalyMiddleware(MiddlewareMixin):
             return None
         request._start_time = time.time()
         ip = get_ip(request)
+        if is_ip_exempted(ip):
+            return None
         if BlacklistManager.is_blocked(ip):
             return JsonResponse({"error": "blocked"}, status=403)
         return None
@@ -166,8 +174,9 @@ class AIAnomalyMiddleware(MiddlewareMixin):
         feats = [path_len, kw_hits, resp_time, status_idx, burst_count, total_404]
         X = np.array(feats, dtype=float).reshape(1, -1)
         if self.model.predict(X)[0] == -1:
-            BlacklistManager.block(ip, "AI anomaly")
-            return JsonResponse({"error": "blocked"}, status=403)
+            if not is_ip_exempted(ip):
+                BlacklistManager.block(ip, "AI anomaly")
+                return JsonResponse({"error": "blocked"}, status=403)
 
         data.append((now, request.path, response.status_code, resp_time))
         data = [d for d in data if now - d[0] < self.WINDOW]
@@ -187,8 +196,9 @@ class HoneypotMiddleware(MiddlewareMixin):
         trap = request.POST.get(getattr(settings, "AIWAF_HONEYPOT_FIELD", "hp_field"), "")
         if trap:
             ip = get_ip(request)
-            BlacklistManager.block(ip, "HONEYPOT triggered")
-            return JsonResponse({"error": "bot_detected"}, status=403)
+            if not is_ip_exempted(ip):
+                BlacklistManager.block(ip, "HONEYPOT triggered")
+                return JsonResponse({"error": "bot_detected"}, status=403)
         return None
 
 
@@ -211,5 +221,6 @@ class UUIDTamperMiddleware(MiddlewareMixin):
                 except (ValueError, TypeError):
                     continue
 
-        BlacklistManager.block(ip, "UUID tampering")
-        return JsonResponse({"error": "blocked"}, status=403)
+        if not is_ip_exempted(ip):
+            BlacklistManager.block(ip, "UUID tampering")
+            return JsonResponse({"error": "blocked"}, status=403)

aiwaf/models.py

@@ -33,4 +33,14 @@ class DynamicKeyword(models.Model):
     last_updated = models.DateTimeField(auto_now=True)
 
     class Meta:
-        ordering = ['-count']
+        ordering = ['-count']
+
+
+# Model to store IP addresses that are exempt from blacklisting
+class IPExemption(models.Model):
+    ip_address = models.GenericIPAddressField(unique=True, db_index=True)
+    reason     = models.CharField(max_length=100, blank=True, default="")
+    created_at = models.DateTimeField(auto_now_add=True)
+
+    def __str__(self):
+        return f"{self.ip_address} (Exempted: {self.reason})"

aiwaf/trainer.py

@@ -2,23 +2,23 @@ import os
 import glob
 import gzip
 import re
+import joblib
+
 from datetime import datetime
 from collections import defaultdict, Counter
 
 import pandas as pd
 from sklearn.ensemble import IsolationForest
-import joblib
 
 from django.conf import settings
 from django.apps import apps
 from django.db.models import F
-from django.urls import get_resolver
 
-
-LOG_PATH = settings.AIWAF_ACCESS_LOG
+# ─────────── Configuration ───────────
+LOG_PATH   = settings.AIWAF_ACCESS_LOG
 MODEL_PATH = os.path.join(os.path.dirname(__file__), "resources", "model.pkl")
 
-STATIC_KW = [".php", "xmlrpc", "wp-", ".env", ".git", ".bak", "conflg", "shell", "filemanager"]
+STATIC_KW  = [".php", "xmlrpc", "wp-", ".env", ".git", ".bak", "conflg", "shell", "filemanager"]
 STATUS_IDX = ["200", "403", "404", "500"]
 
 _LOG_RX = re.compile(
@@ -26,112 +26,114 @@ _LOG_RX = re.compile(
     r'(\d{3}).*?"(.*?)" "(.*?)".*?response-time=(\d+\.\d+)'
 )
 
+
 BlacklistEntry = apps.get_model("aiwaf", "BlacklistEntry")
 DynamicKeyword = apps.get_model("aiwaf", "DynamicKeyword")
+IPExemption = apps.get_model("aiwaf", "IPExemption")
+
 
-def is_exempt_path(path):
+def is_exempt_path(path: str) -> bool:
     path = path.lower()
-    exempt_paths = getattr(settings, "AIWAF_EXEMPT_PATHS", [])
-    for exempt in exempt_paths:
+    for exempt in getattr(settings, "AIWAF_EXEMPT_PATHS", []):
         if path == exempt or path.startswith(exempt.rstrip("/") + "/"):
             return True
     return False
 
-def path_exists_in_django(path):
+
+def path_exists_in_django(path: str) -> bool:
     from django.urls import get_resolver
-    from django.urls.resolvers import URLPattern, URLResolver
+    from django.urls.resolvers import URLResolver
 
-    path = path.split("?")[0].lstrip("/")
+    candidate = path.split("?")[0].lstrip("/")
     try:
-        get_resolver().resolve(f"/{path}")
+        get_resolver().resolve(f"/{candidate}")
         return True
     except:
         pass
-    parts = path.split("/")
-    root_resolver = get_resolver()
-    for pattern in root_resolver.url_patterns:
-        if isinstance(pattern, URLResolver):
-            prefix = pattern.pattern.describe().strip("^/")
-            if prefix and path.startswith(prefix):
+
+    root = get_resolver()
+    for p in root.url_patterns:
+        if isinstance(p, URLResolver):
+            prefix = p.pattern.describe().strip("^/")
+            if prefix and candidate.startswith(prefix):
                 return True
     return False
 
-def remove_exempt_keywords():
-    exempt_paths = getattr(settings, "AIWAF_EXEMPT_PATHS", [])
-    exempt_tokens = set()
-
-    for path in exempt_paths:
-        path = path.strip("/").lower()
-        segments = re.split(r"\W+", path)
-        exempt_tokens.update(seg for seg in segments if len(seg) > 3)
 
+def remove_exempt_keywords() -> None:
+    exempt_tokens = set()
+    for path in getattr(settings, "AIWAF_EXEMPT_PATHS", []):
+        for seg in re.split(r"\W+", path.strip("/").lower()):
+            if len(seg) > 3:
+                exempt_tokens.add(seg)
     if exempt_tokens:
-        deleted_count, _ = DynamicKeyword.objects.filter(keyword__in=exempt_tokens).delete()
-        print(f"Removed {deleted_count} dynamic keywords that are now exempt: {list(exempt_tokens)}")
+        DynamicKeyword.objects.filter(keyword__in=exempt_tokens).delete()
+
 
-def _read_all_logs():
+def _read_all_logs() -> list[str]:
     lines = []
     if LOG_PATH and os.path.exists(LOG_PATH):
         with open(LOG_PATH, "r", errors="ignore") as f:
             lines.extend(f.readlines())
-    for path in sorted(glob.glob(f"{LOG_PATH}.*")):
-        opener = gzip.open if path.endswith(".gz") else open
+    for p in sorted(glob.glob(f"{LOG_PATH}.*")):
+        opener = gzip.open if p.endswith(".gz") else open
         try:
-            with opener(path, "rt", errors="ignore") as f:
+            with opener(p, "rt", errors="ignore") as f:
                 lines.extend(f.readlines())
         except OSError:
             continue
     return lines
 
 
-def _parse(line):
+def _parse(line: str) -> dict | None:
     m = _LOG_RX.search(line)
     if not m:
         return None
-    ip, ts_str, path, status, ref, ua, rt = m.groups()
+    ip, ts_str, path, status, *_ , rt = m.groups()
     try:
         ts = datetime.strptime(ts_str.split()[0], "%d/%b/%Y:%H:%M:%S")
     except ValueError:
         return None
     return {
-        "ip": ip,
-        "timestamp": ts,
-        "path": path,
-        "status": status,
-        "ua": ua,
+        "ip":            ip,
+        "timestamp":     ts,
+        "path":          path,
+        "status":        status,
         "response_time": float(rt),
     }
 
 
-
-def train():
+def train() -> None:
     remove_exempt_keywords()
+    # Remove any IPs in IPExemption from the blacklist
+    exempt_ips = set(IPExemption.objects.values_list("ip_address", flat=True))
+    if exempt_ips:
+        BlacklistEntry.objects.filter(ip_address__in=exempt_ips).delete()
     raw_lines = _read_all_logs()
     if not raw_lines:
         print("No log lines found – check AIWAF_ACCESS_LOG setting.")
         return
+
     parsed = []
-    ip_404 = defaultdict(int)
+    ip_404   = defaultdict(int)
     ip_times = defaultdict(list)
-    for ln in raw_lines:
-        rec = _parse(ln)
+
+    for line in raw_lines:
+        rec = _parse(line)
         if not rec:
             continue
         parsed.append(rec)
         ip_times[rec["ip"]].append(rec["timestamp"])
         if rec["status"] == "404":
             ip_404[rec["ip"]] += 1
-    blocked_404 = []
+
+    # 3. Optional immediate 404‐flood blocking
     for ip, count in ip_404.items():
         if count >= 6:
-            obj, created = BlacklistEntry.objects.get_or_create(
+            BlacklistEntry.objects.get_or_create(
                 ip_address=ip,
                 defaults={"reason": "Excessive 404s (≥6)"}
             )
-            if created:
-                blocked_404.append(ip)
-    if blocked_404:
-        print(f"Blocked {len(blocked_404)} IPs for 404 flood: {blocked_404}")
 
     feature_dicts = []
     for r in parsed:
@@ -140,60 +142,55 @@ def train():
             1 for t in ip_times[ip]
             if (r["timestamp"] - t).total_seconds() <= 10
         )
-        total404 = ip_404[ip]
-        is_known_path = path_exists_in_django(r["path"])
-        kw_hits = 0
-        if not is_known_path and not is_exempt_path(r["path"]):
+        total404   = ip_404[ip]
+        known_path = path_exists_in_django(r["path"])
+        kw_hits    = 0
+        if not known_path and not is_exempt_path(r["path"]):
             kw_hits = sum(k in r["path"].lower() for k in STATIC_KW)
+
         status_idx = STATUS_IDX.index(r["status"]) if r["status"] in STATUS_IDX else -1
+
         feature_dicts.append({
-            "ip": ip,
-            "path_len": len(r["path"]),
-            "kw_hits": kw_hits,
-            "resp_time": r["response_time"],
-            "status_idx": status_idx,
-            "burst_count": burst,
-            "total_404": total404,
+            "ip":           ip,
+            "path_len":     len(r["path"]),
+            "kw_hits":      kw_hits,
+            "resp_time":    r["response_time"],
+            "status_idx":   status_idx,
+            "burst_count":  burst,
+            "total_404":    total404,
         })
 
     if not feature_dicts:
         print("⚠️ Nothing to train on – no valid log entries.")
         return
+
     df = pd.DataFrame(feature_dicts)
     feature_cols = [c for c in df.columns if c != "ip"]
     X = df[feature_cols].astype(float).values
     model = IsolationForest(contamination=0.01, random_state=42)
     model.fit(X)
+
     os.makedirs(os.path.dirname(MODEL_PATH), exist_ok=True)
     joblib.dump(model, MODEL_PATH)
     print(f"Model trained on {len(X)} samples → {MODEL_PATH}")
     preds = model.predict(X)
-    anomalous_ips = set(df.loc[preds == -1, 'ip'])
-    blocked_anom = []
+    anomalous_ips = set(df.loc[preds == -1, "ip"])
     for ip in anomalous_ips:
-        obj, created = BlacklistEntry.objects.get_or_create(
+        BlacklistEntry.objects.get_or_create(
             ip_address=ip,
             defaults={"reason": "Anomalous behavior"}
         )
-        if created:
-            blocked_anom.append(ip)
-    if blocked_anom:
-        print(f"🚫 Blocked {len(blocked_anom)} anomalous IPs: {blocked_anom}")
+
     tokens = Counter()
     for r in parsed:
-        if r["status"].startswith(("4", "5")) and not path_exists_in_django(r["path"]):
+        if (r["status"].startswith(("4", "5"))
+            and not path_exists_in_django(r["path"])):
             for seg in re.split(r"\W+", r["path"].lower()):
                 if len(seg) > 3 and seg not in STATIC_KW:
                     tokens[seg] += 1
 
-    top_tokens = tokens.most_common(10)
-    for kw, cnt in top_tokens:
+    for kw, cnt in tokens.most_common(10):
         obj, _ = DynamicKeyword.objects.get_or_create(keyword=kw)
         DynamicKeyword.objects.filter(pk=obj.pk).update(count=F("count") + cnt)
 
-    print(f" DynamicKeyword DB updated with top tokens: {[kw for kw, _ in top_tokens]}")
-
-
-
-if __name__ == "__main__":
-    train()
+    print(f"DynamicKeyword DB updated with top tokens: {[kw for kw, _ in tokens.most_common(10)]}")

{aiwaf-0.1.7.6.dist-info → aiwaf-0.1.7.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aiwaf
-Version: 0.1.7.6
+Version: 0.1.7.8
 Summary: AI-powered Web Application Firewall
 Home-page: https://github.com/aayushgauba/aiwaf
 Author: Aayush Gauba
@@ -9,6 +9,11 @@ License: MIT
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: Django>=3.2
+Requires-Dist: numpy>=1.21
+Requires-Dist: pandas>=1.3
+Requires-Dist: scikit-learn>=1.0
+Requires-Dist: joblib>=1.1
 Dynamic: author
 Dynamic: home-page
 Dynamic: license-file
@@ -78,13 +83,32 @@ aiwaf/
 - **UUID Tampering Protection**  
   Blocks guessed or invalid UUIDs that don’t resolve to real models.
 
-- **Exempt Path Awareness**  
-  Fully respects `AIWAF_EXEMPT_PATHS` across all modules — exempt paths are:
+
+**Exempt Path & IP Awareness**
+
+**Exempt Paths:**
+Set `AIWAF_EXEMPT_PATHS` in your Django `settings.py` (not in your code). Fully respects this setting across all modules — exempt paths are:
   - Skipped from keyword learning
   - Immune to AI blocking
   - Ignored in log training
   - Cleaned from `DynamicKeyword` model automatically
 
+**Exempt IPs:**
+You can exempt specific IP addresses from all blocking and blacklisting logic. Exempted IPs will:
+  - Never be added to the blacklist (even if they trigger rules)
+  - Be automatically removed from the blacklist during retraining
+  - Bypass all block/deny logic in middleware
+
+### Managing Exempt IPs
+
+Add an IP to the exemption list using the management command:
+
+```bash
+python manage.py add_ipexemption <ip-address> --reason "optional reason"
+```
+
+This will ensure the IP is never blocked by AI‑WAF. You can also manage exemptions via the Django admin interface.
+
 - **Daily Retraining**  
   Reads rotated logs, auto‑blocks 404 floods, retrains the IsolationForest, updates `model.pkl`, and evolves the keyword DB.
 
@@ -125,7 +149,7 @@ AIWAF_RATE_MAX           = 20         # max requests per window
 AIWAF_RATE_FLOOD         = 10         # flood threshold
 AIWAF_WINDOW_SECONDS     = 60         # anomaly detection window
 AIWAF_FILE_EXTENSIONS    = [".php", ".asp", ".jsp"]
-AIWAF_EXEMPT_PATHS       = [          # optional but highly recommended
+AIWAF_EXEMPT_PATHS = [          # optional but highly recommended
     "/favicon.ico",
     "/robots.txt",
     "/static/",

{aiwaf-0.1.7.6.dist-info → aiwaf-0.1.7.8.dist-info}/RECORD

@@ -1,19 +1,20 @@
 aiwaf/__init__.py,sha256=nQFpJ1YpX48snzLjEQCf8zD2YNh8v0b_kPTrXx8uBYc,46
 aiwaf/apps.py,sha256=nCez-Ptlv2kaEk5HenA8b1pATz1VfhrHP1344gwcY1A,142
 aiwaf/blacklist_manager.py,sha256=sM6uTH7zD6MOPGb0kzqV2aFut2vxKgft_UVeRJr7klw,392
-aiwaf/middleware.py,sha256=kH77E1xWVIjQF6frUGM6kdoz-gZXGAh43Fj-2hPEbSM,7990
-aiwaf/models.py,sha256=8au1umopgCo0lthztTTRrYRJQUM7uX8eAeXgs3z45K4,1282
+aiwaf/middleware.py,sha256=8iMoJXGQ86HtvVbmAJa7ykZG-QRdXMIfYmD-hbF8TFg,8456
+aiwaf/models.py,sha256=XaG1pd_oZu3y-fw66u4wblGlWcUY9gvsTNKGD0kQk7Y,1672
 aiwaf/storage.py,sha256=bxCILzzvA1-q6nwclRE8WrfoRhe25H4VrsQDf0hl_lY,1903
-aiwaf/trainer.py,sha256=ir5kFTeLQuhMd2h094ct03Wr-rNZsX-mZHwjLx29F54,6422
+aiwaf/trainer.py,sha256=Xs_AuA7RCa1oyo5-lJYlnRYUiaq-HY2KXAviAdiGnzU,6217
 aiwaf/utils.py,sha256=RkEUWhhHy6tOk7V0UYv3cN4xhOR_7aBy9bjhwuV2cdA,1436
 aiwaf/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aiwaf/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+aiwaf/management/commands/add_ipexemption.py,sha256=LWN21_ydqSjU3_hUnkou4Ciyrk_479zLvcKdWm8hkC0,988
 aiwaf/management/commands/detect_and_train.py,sha256=-o-LZ7QZ5GeJPCekryox1DGXKMmFEkwwrcDsiM166K0,269
-aiwaf/resources/model.pkl,sha256=rCCXH38SJrnaOba2WZrU1LQVzWT34x6bTVkq20XJU-Q,1091129
+aiwaf/resources/model.pkl,sha256=5t6h9BX8yoh2xct85MXOO60jdlWyg1APskUOW0jZE1Y,1288265
 aiwaf/templatetags/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aiwaf/templatetags/aiwaf_tags.py,sha256=1KGqeioYmgKACDUiPkykSqI7DLQ6-Ypy1k00weWj9iY,399
-aiwaf-0.1.7.6.dist-info/licenses/LICENSE,sha256=Ir8PX4dxgAcdB0wqNPIkw84fzIIRKE75NoUil9RX0QU,1069
-aiwaf-0.1.7.6.dist-info/METADATA,sha256=wzS_EmYIHPo4JULdOAoVZvWn7Yo2I9qrRkcWkHw-k34,6116
-aiwaf-0.1.7.6.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
-aiwaf-0.1.7.6.dist-info/top_level.txt,sha256=kU6EyjobT6UPCxuWpI_BvcHDG0I2tMgKaPlWzVxe2xI,6
-aiwaf-0.1.7.6.dist-info/RECORD,,
+aiwaf-0.1.7.8.dist-info/licenses/LICENSE,sha256=Ir8PX4dxgAcdB0wqNPIkw84fzIIRKE75NoUil9RX0QU,1069
+aiwaf-0.1.7.8.dist-info/METADATA,sha256=Hthw0o3R1p6JeLo8qfHYKrVCYHB5RgD9lP1zTWlgA8s,6920
+aiwaf-0.1.7.8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+aiwaf-0.1.7.8.dist-info/top_level.txt,sha256=kU6EyjobT6UPCxuWpI_BvcHDG0I2tMgKaPlWzVxe2xI,6
+aiwaf-0.1.7.8.dist-info/RECORD,,

{aiwaf-0.1.7.6.dist-info → aiwaf-0.1.7.8.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.4.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any