aiwaf 0.1.6__py3-none-any.whl → 0.1.7.1__py3-none-any.whl

aiwaf/middleware.py

@@ -5,7 +5,7 @@ import re
 import os
 import numpy as np
 import joblib
-
+from django.db.models import UUIDField
 from collections import defaultdict
 from django.utils.deprecation import MiddlewareMixin
 from django.http import JsonResponse
@@ -18,6 +18,14 @@ from django.urls import get_resolver
 from .blacklist_manager import BlacklistManager
 from .models import DynamicKeyword
 
+def is_exempt_path(path):
+    path = path.lower()
+    exempt_paths = getattr(settings, "AIWAF_EXEMPT_PATHS", [])
+    for exempt in exempt_paths:
+        if path == exempt or path.startswith(exempt.rstrip("/") + "/"):
+            return True
+    return False
+
 MODEL_PATH = getattr(
     settings,
     "AIWAF_MODEL_PATH",
@@ -43,25 +51,32 @@ def get_ip(request):
 class IPAndKeywordBlockMiddleware:
     def __init__(self, get_response):
         self.get_response = get_response
-        self.url_patterns = self._collect_view_paths()
+        self.safe_prefixes = self._collect_safe_prefixes()
 
-    def _collect_view_paths(self):
+    def _collect_safe_prefixes(self):
         resolver = get_resolver()
-        patterns = set()
+        prefixes = set()
 
         def extract(patterns_list, prefix=""):
             for p in patterns_list:
-                if hasattr(p, "url_patterns"):
+                if hasattr(p, "url_patterns"):  # include()
+                    full_prefix = (prefix + str(p.pattern)).strip("^/").split("/")[0]
+                    prefixes.add(full_prefix)
                     extract(p.url_patterns, prefix + str(p.pattern))
                 else:
                     pat = (prefix + str(p.pattern)).strip("^$")
-                    patterns.add(pat)
+                    path_parts = pat.strip("/").split("/")
+                    if path_parts:
+                        prefixes.add(path_parts[0])
         extract(resolver.url_patterns)
-        return patterns
+        return prefixes
 
     def __call__(self, request):
+        raw_path = request.path.lower()
+        if is_exempt_path(raw_path):
+            return self.get_response(request)
         ip = get_ip(request)
-        path = request.path.lower()
+        path = raw_path.lstrip("/")
         if BlacklistManager.is_blocked(ip):
             return JsonResponse({"error": "blocked"}, status=403)
         segments = [seg for seg in re.split(r"\W+", path) if len(seg) > 3]
@@ -74,8 +89,10 @@ class IPAndKeywordBlockMiddleware:
             .values_list("keyword", flat=True)[: getattr(settings, "AIWAF_DYNAMIC_TOP_N", 10)]
         )
         all_kw = set(STATIC_KW) | set(dynamic_top)
-        safe_kw = {kw for kw in all_kw if any(kw in pat for pat in self.url_patterns)}
-        suspicious_kw = all_kw - safe_kw
+        suspicious_kw = {
+            kw for kw in all_kw
+            if not any(path.startswith(prefix) for prefix in self.safe_prefixes if prefix)
+        }
         for seg in segments:
             if seg in suspicious_kw:
                 BlacklistManager.block(ip, f"Keyword block: {seg}")
@@ -93,6 +110,8 @@ class RateLimitMiddleware:
         self.logs = defaultdict(list)
 
     def __call__(self, request):
+        if is_exempt_path(request.path):
+            return self.get_response(request)
         ip  = get_ip(request)
         now = time.time()
         recs = [t for t in self.logs[ip] if now - t < self.WINDOW]
@@ -113,6 +132,8 @@ class AIAnomalyMiddleware(MiddlewareMixin):
     TOP_N  = getattr(settings, "AIWAF_DYNAMIC_TOP_N", 10)
 
     def process_request(self, request):
+        if is_exempt_path(request.path):
+            return None
         ip = get_ip(request)
         if BlacklistManager.is_blocked(ip):
             return JsonResponse({"error": "blocked"}, status=403)
@@ -120,12 +141,9 @@ class AIAnomalyMiddleware(MiddlewareMixin):
         now = time.time()
         key = f"aiwaf:{ip}"
         data = cache.get(key, [])
-        # TODO: you may want to capture real status & response_time in process_response
         data.append((now, request.path, 0, 0.0))
         data = [d for d in data if now - d[0] < self.WINDOW]
         cache.set(key, data, timeout=self.WINDOW)
-
-        # update dynamic‐keyword counts
         for seg in re.split(r"\W+", request.path.lower()):
             if len(seg) > 3:
                 obj, _ = DynamicKeyword.objects.get_or_create(keyword=seg)
@@ -133,8 +151,6 @@ class AIAnomalyMiddleware(MiddlewareMixin):
 
         if len(data) < 5:
             return None
-
-        # pull top‐N dynamic tokens
         top_dynamic = list(
             DynamicKeyword.objects
             .order_by("-count")
@@ -159,6 +175,8 @@ class AIAnomalyMiddleware(MiddlewareMixin):
 
 class HoneypotMiddleware(MiddlewareMixin):
     def process_view(self, request, view_func, view_args, view_kwargs):
+        if is_exempt_path(request.path):
+            return None
         trap = request.POST.get(getattr(settings, "AIWAF_HONEYPOT_FIELD", "hp_field"), "")
         if trap:
             ip = get_ip(request)
@@ -169,6 +187,8 @@ class HoneypotMiddleware(MiddlewareMixin):
 
 class UUIDTamperMiddleware(MiddlewareMixin):
     def process_view(self, request, view_func, view_args, view_kwargs):
+        if is_exempt_path(request.path):
+            return None
         uid = view_kwargs.get("uuid")
         if not uid:
             return None
@@ -177,8 +197,12 @@ class UUIDTamperMiddleware(MiddlewareMixin):
         app_label = view_func.__module__.split(".")[0]
         app_cfg   = apps.get_app_config(app_label)
         for Model in app_cfg.get_models():
-            if Model.objects.filter(pk=uid).exists():
-                return None
+            if isinstance(Model._meta.pk, UUIDField):
+                try:
+                    if Model.objects.filter(pk=uid).exists():
+                        return None
+                except (ValueError, TypeError):
+                    continue
 
         BlacklistManager.block(ip, "UUID tampering")
         return JsonResponse({"error": "blocked"}, status=403)

aiwaf/trainer.py

@@ -12,6 +12,8 @@ import joblib
 from django.conf import settings
 from django.apps import apps
 from django.db.models import F
+from django.urls import get_resolver
+
 
 LOG_PATH = settings.AIWAF_ACCESS_LOG
 MODEL_PATH = os.path.join(os.path.dirname(__file__), "resources", "model.pkl")
@@ -27,6 +29,45 @@ _LOG_RX = re.compile(
 BlacklistEntry = apps.get_model("aiwaf", "BlacklistEntry")
 DynamicKeyword = apps.get_model("aiwaf", "DynamicKeyword")
 
+def is_exempt_path(path):
+    path = path.lower()
+    exempt_paths = getattr(settings, "AIWAF_EXEMPT_PATHS", [])
+    for exempt in exempt_paths:
+        if path == exempt or path.startswith(exempt.rstrip("/") + "/"):
+            return True
+    return False
+
+def path_exists_in_django(path):
+    from django.urls import get_resolver
+    from django.urls.resolvers import URLPattern, URLResolver
+
+    path = path.split("?")[0].lstrip("/")
+    try:
+        get_resolver().resolve(f"/{path}")
+        return True
+    except:
+        pass
+    parts = path.split("/")
+    root_resolver = get_resolver()
+    for pattern in root_resolver.url_patterns:
+        if isinstance(pattern, URLResolver):
+            prefix = pattern.pattern.describe().strip("^/")
+            if prefix and path.startswith(prefix):
+                return True
+    return False
+
+def remove_exempt_keywords():
+    exempt_paths = getattr(settings, "AIWAF_EXEMPT_PATHS", [])
+    exempt_tokens = set()
+
+    for path in exempt_paths:
+        path = path.strip("/").lower()
+        segments = re.split(r"\W+", path)
+        exempt_tokens.update(seg for seg in segments if len(seg) > 3)
+
+    if exempt_tokens:
+        deleted_count, _ = DynamicKeyword.objects.filter(keyword__in=exempt_tokens).delete()
+        print(f"Removed {deleted_count} dynamic keywords that are now exempt: {list(exempt_tokens)}")
 
 def _read_all_logs():
     lines = []
@@ -62,10 +103,12 @@ def _parse(line):
     }
 
 
+
 def train():
+    remove_exempt_keywords()
     raw_lines = _read_all_logs()
     if not raw_lines:
-        print(" No log lines found – check AIWAF_ACCESS_LOG setting.")
+        print("No log lines found – check AIWAF_ACCESS_LOG setting.")
         return
     parsed = []
     ip_404 = defaultdict(int)
@@ -89,6 +132,7 @@ def train():
                 blocked_404.append(ip)
     if blocked_404:
         print(f"Blocked {len(blocked_404)} IPs for 404 flood: {blocked_404}")
+
     feature_dicts = []
     for r in parsed:
         ip = r["ip"]
@@ -97,7 +141,10 @@ def train():
             if (r["timestamp"] - t).total_seconds() <= 10
         )
         total404 = ip_404[ip]
-        kw_hits = sum(k in r["path"].lower() for k in STATIC_KW)
+        is_known_path = path_exists_in_django(r["path"])
+        kw_hits = 0
+        if not is_known_path and not is_exempt_path(r["path"]):
+            kw_hits = sum(k in r["path"].lower() for k in STATIC_KW)
         status_idx = STATUS_IDX.index(r["status"]) if r["status"] in STATUS_IDX else -1
         feature_dicts.append({
             "ip": ip,
@@ -112,7 +159,6 @@ def train():
     if not feature_dicts:
         print("⚠️ Nothing to train on – no valid log entries.")
         return
-
     df = pd.DataFrame(feature_dicts)
     feature_cols = [c for c in df.columns if c != "ip"]
     X = df[feature_cols].astype(float).values
@@ -120,8 +166,8 @@ def train():
     model.fit(X)
     os.makedirs(os.path.dirname(MODEL_PATH), exist_ok=True)
     joblib.dump(model, MODEL_PATH)
-    print(f"✅ Model trained on {len(X)} samples → {MODEL_PATH}")
-    preds = model.predict(X)  # -1 for outliers
+    print(f"Model trained on {len(X)} samples → {MODEL_PATH}")
+    preds = model.predict(X)
     anomalous_ips = set(df.loc[preds == -1, 'ip'])
     blocked_anom = []
     for ip in anomalous_ips:
@@ -132,20 +178,22 @@ def train():
         if created:
             blocked_anom.append(ip)
     if blocked_anom:
-        print(f" Blocked {len(blocked_anom)} anomalous IPs: {blocked_anom}")
-
+        print(f"🚫 Blocked {len(blocked_anom)} anomalous IPs: {blocked_anom}")
     tokens = Counter()
     for r in parsed:
-        if r["status"].startswith(("4", "5")):
+        if r["status"].startswith(("4", "5")) and not path_exists_in_django(r["path"]):
             for seg in re.split(r"\W+", r["path"].lower()):
                 if len(seg) > 3 and seg not in STATIC_KW:
                     tokens[seg] += 1
+
     top_tokens = tokens.most_common(10)
     for kw, cnt in top_tokens:
         obj, _ = DynamicKeyword.objects.get_or_create(keyword=kw)
         DynamicKeyword.objects.filter(pk=obj.pk).update(count=F("count") + cnt)
-    print(f"DynamicKeyword DB updated with top tokens: {[kw for kw, _ in top_tokens]}")
+
+    print(f" DynamicKeyword DB updated with top tokens: {[kw for kw, _ in top_tokens]}")
+
 
 
 if __name__ == "__main__":
-    train()
+    train()

{aiwaf-0.1.6.dist-info → aiwaf-0.1.7.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aiwaf
-Version: 0.1.6
+Version: 0.1.7.1
 Summary: AI-powered Web Application Firewall
 Home-page: https://github.com/aayushgauba/aiwaf
 Author: Aayush Gauba
@@ -15,14 +15,14 @@ Dynamic: license-file
 Dynamic: requires-python
 
 
-# AI‑WAF 
+# AI‑WAF
 
 > A self‑learning, Django‑friendly Web Application Firewall  
-> with rate‑limiting, anomaly detection, honeypots, UUID‑tamper protection, dynamic keyword extraction, file‑extension probing detection, and daily retraining.
+> with rate‑limiting, anomaly detection, honeypots, UUID‑tamper protection, dynamic keyword extraction, file‑extension probing detection, exempt path awareness, and daily retraining.
 
 ---
 
-## Package Structure
+## 📁 Package Structure
 
 ```
 aiwaf/
@@ -44,7 +44,7 @@ aiwaf/
 
 ---
 
-## Features
+## 🚀 Features
 
 - **IP Blocklist**  
   Instantly blocks suspicious IPs (supports CSV fallback or Django model).
@@ -53,7 +53,7 @@ aiwaf/
   Sliding‑window blocks flooders (> `AIWAF_RATE_MAX` per `AIWAF_RATE_WINDOW`), then blacklists them.
 
 - **AI Anomaly Detection**  
-  IsolationForest on features:
+  IsolationForest trained on:
   - Path length  
   - Keyword hits (static + dynamic)  
   - Response time  
@@ -61,34 +61,28 @@ aiwaf/
   - Burst count  
   - Total 404s  
 
-- **Dynamic Keyword Extraction**  
-  Every retrain: top 10 most frequent “words” from 4xx/5xx paths are appended to your malicious keyword set.
+- **Dynamic Keyword Extraction & Cleanup**  
+  - Every retrain adds top 10 keyword segments from 4xx/5xx paths  
+  - **If a path is added to `AIWAF_EXEMPT_PATHS`, its keywords are automatically removed from the database**
 
 - **File‑Extension Probing Detection**  
-  Tracks repeated 404s on common web‑extensions (e.g. `.php`, `.asp`) and auto‑blocks after a burst.
+  Tracks repeated 404s on common extensions (e.g. `.php`, `.asp`) and blocks IPs.
 
 - **Honeypot Field**  
-  Hidden form field (via template tag) that bots fill → instant block.
+  Hidden field for bot detection → IP blacklisted on fill.
 
 - **UUID Tampering Protection**  
-  Any `<uuid:…>` URL that doesn’t map to **any** model in its Django app gets blocked.
+  Blocks guessed or invalid UUIDs that don’t resolve to real models.
 
-- **Daily Retraining**  
-  Reads rotated/gzipped logs, auto‑blocks 404 floods (≥6), retrains the model, updates `model.pkl` + `dynamic_keywords.json`.
-
----
-
-## Installation
-
-```bash
-# From PyPI
-pip install aiwaf
+- **Exempt Path Awareness**  
+  Fully respects `AIWAF_EXEMPT_PATHS` across all modules — exempt paths are:
+  - Skipped from keyword learning
+  - Immune to AI blocking
+  - Ignored in log training
+  - Cleaned from `DynamicKeyword` model automatically
 
-# Or for local development
-git clone https://github.com/aayushgauba/aiwaf.git
-cd aiwaf
-pip install -e .
-```
+- **Daily Retraining**  
+  Reads rotated logs, auto‑blocks 404 floods, retrains the IsolationForest, updates `model.pkl`, and evolves the keyword DB.
 
 ---
 
@@ -96,33 +90,51 @@ pip install -e .
 
 ```python
 INSTALLED_APPS += ["aiwaf"]
+```
 
 ### Database Setup
 
-After adding `aiwaf` to your `INSTALLED_APPS`, create the necessary tables for the IP‐blacklist and dynamic‐keyword models:
+After adding `aiwaf` to your `INSTALLED_APPS`, run the following to create the necessary tables:
 
 ```bash
 python manage.py makemigrations aiwaf
 python manage.py migrate
+```
 
-# Required
+---
+
+### Required
+
+```python
 AIWAF_ACCESS_LOG = "/var/log/nginx/access.log"
+```
+
+---
+
+### Optional (defaults shown)
 
-# Optional (defaults shown)
+```python
 AIWAF_MODEL_PATH         = BASE_DIR / "aiwaf" / "resources" / "model.pkl"
 AIWAF_HONEYPOT_FIELD     = "hp_field"
 AIWAF_RATE_WINDOW        = 10         # seconds
-AIWAF_RATE_MAX           = 20         # max reqs/window
+AIWAF_RATE_MAX           = 20         # max requests per window
 AIWAF_RATE_FLOOD         = 10         # flood threshold
-AIWAF_WINDOW_SECONDS     = 60         # anomaly window
-AIWAF_FILE_EXTENSIONS    = [".php", ".asp", ".jsp"]  # 404‑burst tracked extensions
+AIWAF_WINDOW_SECONDS     = 60         # anomaly detection window
+AIWAF_FILE_EXTENSIONS    = [".php", ".asp", ".jsp"]
+AIWAF_EXEMPT_PATHS       = [          # optional but highly recommended
+    "/favicon.ico",
+    "/robots.txt",
+    "/static/",
+    "/media/",
+    "/health/",
+]
 ```
 
-> **Note:** You no longer need to define `AIWAF_MALICIOUS_KEYWORDS` or `AIWAF_STATUS_CODES` in your settings — they’re built in and evolve dynamically.
+> **Note:** You no longer need to define `AIWAF_MALICIOUS_KEYWORDS` or `AIWAF_STATUS_CODES` — they evolve dynamically.
 
 ---
 
-## Middleware Setup
+## 🧱 Middleware Setup
 
 Add in **this** order to your `MIDDLEWARE` list:
 
@@ -139,7 +151,7 @@ MIDDLEWARE = [
 
 ---
 
-## Honeypot Field (in your template)
+## 🕵️ Honeypot Field (in your template)
 
 ```django
 {% load aiwaf_tags %}
@@ -156,22 +168,23 @@ MIDDLEWARE = [
 
 ---
 
-## Running Detection & Training
+## 🔁 Running Detection & Training
 
 ```bash
 python manage.py detect_and_train
 ```
 
-**What happens:**
-1. Read access logs
+### What happens:
+1. Read access logs (incl. rotated or gzipped)
 2. Auto‑block IPs with ≥ 6 total 404s
 3. Extract features & train IsolationForest
 4. Save `model.pkl`
 5. Extract top 10 dynamic keywords from 4xx/5xx
+6. Remove any keywords associated with newly exempt paths
 
 ---
 
-## How It Works
+## 🧠 How It Works
 
 | Middleware                         | Purpose                                                         |
 |------------------------------------|-----------------------------------------------------------------|
@@ -180,15 +193,16 @@ python manage.py detect_and_train
 | AIAnomalyMiddleware                | ML‑driven behavior analysis + block on anomaly                  |
 | HoneypotMiddleware                 | Detects bots filling hidden inputs in forms                     |
 | UUIDTamperMiddleware               | Blocks guessed/nonexistent UUIDs across all models in an app    |
+
 ---
 
-## License
+## 📄 License
 
 This project is licensed under the **MIT License**. See the [LICENSE](LICENSE) file for details.
 
 ---
 
-## Credits
+## 👤 Credits
 
 **AI‑WAF** by [Aayush Gauba](https://github.com/aayushgauba)  
 > “Let your firewall learn and evolve — keep your site a fortress.”

{aiwaf-0.1.6.dist-info → aiwaf-0.1.7.1.dist-info}/RECORD

@@ -1,10 +1,10 @@
 aiwaf/__init__.py,sha256=nQFpJ1YpX48snzLjEQCf8zD2YNh8v0b_kPTrXx8uBYc,46
 aiwaf/apps.py,sha256=nCez-Ptlv2kaEk5HenA8b1pATz1VfhrHP1344gwcY1A,142
 aiwaf/blacklist_manager.py,sha256=sM6uTH7zD6MOPGb0kzqV2aFut2vxKgft_UVeRJr7klw,392
-aiwaf/middleware.py,sha256=04AbNgkwLMaYSiuEtw59A-O02tt4cqaKmP7XDNlkIG0,6359
+aiwaf/middleware.py,sha256=LTLHmQYIQ36WwfR9FEPLrmTbYgqxIh4X5Aen4VJ-vN0,7350
 aiwaf/models.py,sha256=8au1umopgCo0lthztTTRrYRJQUM7uX8eAeXgs3z45K4,1282
 aiwaf/storage.py,sha256=bxCILzzvA1-q6nwclRE8WrfoRhe25H4VrsQDf0hl_lY,1903
-aiwaf/trainer.py,sha256=TKWJZzWTg892vdoSGWdCA0i-dKof2b29buWqJUrkr6k,4820
+aiwaf/trainer.py,sha256=ir5kFTeLQuhMd2h094ct03Wr-rNZsX-mZHwjLx29F54,6422
 aiwaf/utils.py,sha256=RkEUWhhHy6tOk7V0UYv3cN4xhOR_7aBy9bjhwuV2cdA,1436
 aiwaf/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aiwaf/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -12,8 +12,8 @@ aiwaf/management/commands/detect_and_train.py,sha256=-o-LZ7QZ5GeJPCekryox1DGXKMm
 aiwaf/resources/model.pkl,sha256=rCCXH38SJrnaOba2WZrU1LQVzWT34x6bTVkq20XJU-Q,1091129
 aiwaf/template_tags/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aiwaf/template_tags/aiwaf_tags.py,sha256=1KGqeioYmgKACDUiPkykSqI7DLQ6-Ypy1k00weWj9iY,399
-aiwaf-0.1.6.dist-info/licenses/LICENSE,sha256=Ir8PX4dxgAcdB0wqNPIkw84fzIIRKE75NoUil9RX0QU,1069
-aiwaf-0.1.6.dist-info/METADATA,sha256=xwayhSMTf_thMyNrS2-E5Wa8S1vXpqiOuwRZiI8-6Pw,5414
-aiwaf-0.1.6.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-aiwaf-0.1.6.dist-info/top_level.txt,sha256=kU6EyjobT6UPCxuWpI_BvcHDG0I2tMgKaPlWzVxe2xI,6
-aiwaf-0.1.6.dist-info/RECORD,,
+aiwaf-0.1.7.1.dist-info/licenses/LICENSE,sha256=Ir8PX4dxgAcdB0wqNPIkw84fzIIRKE75NoUil9RX0QU,1069
+aiwaf-0.1.7.1.dist-info/METADATA,sha256=aO_1D_qSP_s4vKUj60a8VmsFcCLCyhBZii1tpbo3HqE,5790
+aiwaf-0.1.7.1.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
+aiwaf-0.1.7.1.dist-info/top_level.txt,sha256=kU6EyjobT6UPCxuWpI_BvcHDG0I2tMgKaPlWzVxe2xI,6
+aiwaf-0.1.7.1.dist-info/RECORD,,

{aiwaf-0.1.6.dist-info → aiwaf-0.1.7.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (79.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any