aiwaf 0.1.9.2.2__py3-none-any.whl → 0.1.9.2.4__py3-none-any.whl

aiwaf/__init__.py

@@ -1,6 +1,6 @@
 default_app_config = "aiwaf.apps.AiwafConfig"
 
-__version__ = "0.1.9.2.2"
+__version__ = "0.1.9.2.4"
 
 # Note: Middleware classes are available from aiwaf.middleware
 # Import them only when needed to avoid circular imports during Django app loading

aiwaf/middleware.py

@@ -179,17 +179,26 @@ class IPAndKeywordBlockMiddleware:
             def extract_from_pattern(pattern, prefix=""):
                 try:
                     if isinstance(pattern, URLResolver):
-                        # Handle include() patterns
+                        # Handle include() patterns - be permissive for URL prefixes that route to apps
                         namespace = getattr(pattern, 'namespace', None)
                         if namespace:
                             for segment in re.split(r'[._-]', namespace.lower()):
                                 if len(segment) > 2:
                                     keywords.add(segment)
                         
-                        # Extract from the pattern itself
+                        # Extract from the pattern itself - improved logic for include() patterns
                         pattern_str = str(pattern.pattern)
-                        for segment in re.findall(r'([a-zA-Z]\w{2,})', pattern_str):
-                            keywords.add(segment.lower())
+                        # Get literal path segments (not regex parts)
+                        literal_parts = re.findall(r'([a-zA-Z][a-zA-Z0-9_-]*)', pattern_str)
+                        
+                        # For include() patterns, be more permissive since they're routing to existing apps
+                        # The key insight: if someone includes an app's URLs, the prefix is legitimate by design
+                        for part in literal_parts:
+                            if len(part) > 2:
+                                part_lower = part.lower()
+                                # For URLResolver (include patterns), be more permissive
+                                # These are URL prefixes that route to actual app functionality
+                                keywords.add(part_lower)
                         
                         # Recurse into nested patterns
                         for nested_pattern in pattern.url_patterns:
@@ -298,11 +307,15 @@ class IPAndKeywordBlockMiddleware:
         keyword_store = get_keyword_store()
         segments = [seg for seg in re.split(r"\W+", path) if len(seg) > 3]
         
-        # Only learn keywords from non-existent paths or suspicious contexts
-        for seg in segments:
-            if not path_exists or self._is_malicious_context(request, seg):
-                keyword_store.add_keyword(seg)
-            
+        # Smart learning: only learn from suspicious contexts, never from valid paths
+        if not path_exists:  # Only learn from non-existent paths
+            for seg in segments:
+                # Only learn if it's not a legitimate keyword AND in a suspicious context
+                if (seg not in self.legitimate_path_keywords and 
+                    seg not in self.exempt_keywords and
+                    self._is_malicious_context(request, seg)):
+                    keyword_store.add_keyword(seg)
+        
         dynamic_top = keyword_store.get_top_keywords(getattr(settings, "AIWAF_DYNAMIC_TOP_N", 10))
         all_kw = set(STATIC_KW) | set(dynamic_top)
         
@@ -345,7 +358,29 @@ class IPAndKeywordBlockMiddleware:
                 block_reason = f"Inherently suspicious: {seg}"
             
             if is_suspicious:
-                # Additional context check before blocking
+                # Additional context check before blocking - be more conservative with valid paths
+                if path_exists:
+                    # For valid paths, only block if there are VERY strong malicious indicators
+                    very_strong_indicators = [
+                        # Multiple attack patterns in same request
+                        sum([
+                            '../' in request.path, '..\\' in request.path,
+                            any(param in request.GET for param in ['cmd', 'exec', 'system']),
+                            request.path.count('%') > 5,  # Heavy URL encoding
+                            len([s for s in segments if s in self.malicious_keywords]) > 2
+                        ]) >= 2,
+                        
+                        # Obvious attack attempts on valid paths
+                        any(attack in request.path.lower() for attack in [
+                            'union+select', 'drop+table', '<script', 'javascript:',
+                            'onload=', 'onerror=', '${', '{{', 'eval('
+                        ])
+                    ]
+                    
+                    if not any(very_strong_indicators):
+                        continue  # Skip blocking for valid paths without very strong indicators
+                
+                # For non-existent paths or paths with very strong indicators, proceed with blocking
                 if self._is_malicious_context(request, seg) or not path_exists:
                     # Double-check exemption before blocking
                     if not exemption_store.is_exempted(ip):
@@ -405,6 +440,38 @@ class AIAnomalyMiddleware(MiddlewareMixin):
         # Use the safely loaded global MODEL instead of loading again
         self.model = MODEL
 
+    def _is_malicious_context(self, request, keyword):
+        """
+        Determine if a keyword appears in a malicious context.
+        Only learn keywords when we have strong indicators of malicious intent.
+        """
+        # Don't learn from valid Django paths
+        if path_exists_in_django(request.path):
+            return False
+            
+        # Strong malicious indicators
+        malicious_indicators = [
+            # Multiple consecutive suspicious segments
+            len([seg for seg in re.split(r"\W+", request.path) if seg in self.malicious_keywords]) > 1,
+            
+            # Common attack patterns
+            any(pattern in request.path.lower() for pattern in [
+                '../', '..\\', '.env', 'wp-admin', 'phpmyadmin', 'config',
+                'backup', 'database', 'mysql', 'passwd', 'shadow'
+            ]),
+            
+            # Suspicious query parameters
+            any(param in request.GET for param in ['cmd', 'exec', 'system', 'shell']),
+            
+            # Multiple directory traversal attempts
+            request.path.count('../') > 2 or request.path.count('..\\') > 2,
+            
+            # Encoded attack patterns
+            any(encoded in request.path for encoded in ['%2e%2e', '%252e', '%c0%ae']),
+        ]
+        
+        return any(malicious_indicators)
+
     def process_request(self, request):
         # First exemption check - early exit for exempt requests
         if is_exempt(request):

aiwaf/trainer.py

@@ -96,7 +96,7 @@ def get_legitimate_keywords() -> set:
     """Get all legitimate keywords that shouldn't be learned as suspicious"""
     legitimate = set()
     
-    # Common legitimate path segments
+    # Common legitimate path segments - expanded set
     default_legitimate = {
         "profile", "user", "users", "account", "accounts", "settings", "dashboard", 
         "home", "about", "contact", "help", "search", "list", "lists",
@@ -106,7 +106,32 @@ def get_legitimate_keywords() -> set:
         "category", "categories", "tag", "tags", "post", "posts",
         "article", "articles", "blog", "blogs", "news", "item", "items",
         "admin", "administration", "manage", "manager", "control", "panel",
-        "config", "configuration", "option", "options", "preference", "preferences"
+        "config", "configuration", "option", "options", "preference", "preferences",
+        
+        # Django built-in app keywords
+        "contenttypes", "contenttype", "sessions", "session", "messages", "message",
+        "staticfiles", "static", "sites", "site", "flatpages", "flatpage",
+        "redirects", "redirect", "permissions", "permission", "groups", "group",
+        
+        # Common third-party package keywords
+        "token", "tokens", "oauth", "social", "rest", "framework", "cors",
+        "debug", "toolbar", "extensions", "allauth", "crispy", "forms",
+        "channels", "celery", "redis", "cache", "email", "mail",
+        
+        # Common API/web development terms
+        "endpoint", "endpoints", "resource", "resources", "data", "export",
+        "import", "upload", "download", "file", "files", "media", "images",
+        "documents", "reports", "analytics", "stats", "statistics",
+        
+        # Common business/application terms
+        "customer", "customers", "client", "clients", "company", "companies",
+        "department", "departments", "employee", "employees", "team", "teams",
+        "project", "projects", "task", "tasks", "event", "events",
+        "notification", "notifications", "alert", "alerts",
+        
+        # Language/localization
+        "language", "languages", "locale", "locales", "translation", "translations",
+        "en", "fr", "de", "es", "it", "pt", "ru", "ja", "zh", "ko"
     }
     legitimate.update(default_legitimate)
     
@@ -135,60 +160,103 @@ def _extract_django_route_keywords() -> set:
         
         # Extract from app names and labels
         for app_config in apps.get_app_configs():
-            # Add app name and label
+            # Add app name and label - improved parsing
             if app_config.name:
-                for segment in re.split(r'[._-]', app_config.name.lower()):
-                    if len(segment) > 2:
-                        keywords.add(segment)
+                app_parts = app_config.name.lower().replace('-', '_').split('.')
+                for part in app_parts:
+                    for segment in re.split(r'[._-]', part):
+                        if len(segment) > 2:
+                            keywords.add(segment)
             
             if app_config.label and app_config.label != app_config.name:
                 for segment in re.split(r'[._-]', app_config.label.lower()):
                     if len(segment) > 2:
                         keywords.add(segment)
             
-            # Extract from model names in the app
+            # Extract from model names in the app - improved handling
             try:
                 for model in app_config.get_models():
                     model_name = model._meta.model_name.lower()
                     if len(model_name) > 2:
                         keywords.add(model_name)
-                    # Add plural form
-                    if not model_name.endswith('s'):
-                        keywords.add(f"{model_name}s")
+                        # Add plural form
+                        if not model_name.endswith('s'):
+                            keywords.add(f"{model_name}s")
+                    
+                    # Also add verbose names if different
+                    verbose_name = str(model._meta.verbose_name).lower()
+                    verbose_name_plural = str(model._meta.verbose_name_plural).lower()
+                    
+                    for name in [verbose_name, verbose_name_plural]:
+                        for segment in re.split(r'[^a-zA-Z]+', name):
+                            if len(segment) > 2 and segment != model_name:
+                                keywords.add(segment)
             except Exception:
                 continue
         
-        # Extract from URL patterns
+        # Extract from URL patterns - improved extraction
         def extract_from_pattern(pattern, prefix=""):
             try:
                 if isinstance(pattern, URLResolver):
-                    # Handle include() patterns
+                    # Handle include() patterns - check if they include legitimate apps
                     namespace = getattr(pattern, 'namespace', None)
                     if namespace:
                         for segment in re.split(r'[._-]', namespace.lower()):
                             if len(segment) > 2:
                                 keywords.add(segment)
                     
-                    # Extract from the pattern itself
+                    # Extract from the pattern itself - improved logic for include() patterns
                     pattern_str = str(pattern.pattern)
-                    for segment in re.findall(r'([a-zA-Z]\w{2,})', pattern_str):
-                        keywords.add(segment.lower())
+                    # Get literal path segments (not regex parts)
+                    literal_parts = re.findall(r'([a-zA-Z][a-zA-Z0-9_-]*)', pattern_str)
+                    
+                    # Get list of actual Django app names to validate against
+                    app_names = set()
+                    for app_config in apps.get_app_configs():
+                        app_parts = app_config.name.lower().replace('-', '_').split('.')
+                        for part in app_parts:
+                            for segment in re.split(r'[._-]', part):
+                                if len(segment) > 2:
+                                    app_names.add(segment)
+                        if app_config.label:
+                            app_names.add(app_config.label.lower())
+                    
+                    # For include() patterns, be more permissive since they're routing to existing apps
+                    # The key insight: if someone includes an app's URLs, the prefix is legitimate by design
+                    for part in literal_parts:
+                        if len(part) > 2:
+                            part_lower = part.lower()
+                            # For URLResolver (include patterns), be more permissive
+                            # These are URL prefixes that route to actual app functionality
+                            keywords.add(part_lower)
                     
                     # Recurse into nested patterns
-                    for nested_pattern in pattern.url_patterns:
-                        extract_from_pattern(nested_pattern, prefix)
+                    try:
+                        for nested_pattern in pattern.url_patterns:
+                            extract_from_pattern(nested_pattern, prefix)
+                    except:
+                        pass
                 
                 elif isinstance(pattern, URLPattern):
-                    # Extract from URL pattern
+                    # Extract from URL pattern - more comprehensive
                     pattern_str = str(pattern.pattern)
-                    for segment in re.findall(r'([a-zA-Z]\w{2,})', pattern_str):
-                        keywords.add(segment.lower())
+                    literal_parts = re.findall(r'([a-zA-Z][a-zA-Z0-9_-]*)', pattern_str)
+                    for part in literal_parts:
+                        if len(part) > 2:
+                            keywords.add(part.lower())
                     
                     # Extract from view name if available
                     if hasattr(pattern.callback, '__name__'):
                         view_name = pattern.callback.__name__.lower()
                         for segment in re.split(r'[._-]', view_name):
-                            if len(segment) > 2 and segment != 'view':
+                            if len(segment) > 2 and segment not in ['view', 'class', 'function']:
+                                keywords.add(segment)
+                    
+                    # Extract from view class name if it's a class-based view
+                    if hasattr(pattern.callback, 'view_class'):
+                        class_name = pattern.callback.view_class.__name__.lower()
+                        for segment in re.split(r'[._-]', class_name):
+                            if len(segment) > 2 and segment not in ['view', 'class']:
                                 keywords.add(segment)
             
             except Exception:
@@ -203,10 +271,20 @@ def _extract_django_route_keywords() -> set:
         print(f"Warning: Could not extract Django route keywords: {e}")
     
     # Filter out very common/generic words that might be suspicious
+    # Expanded filter list
     filtered_keywords = set()
+    exclude_words = {
+        'www', 'com', 'org', 'net', 'int', 'str', 'obj', 'get', 'set', 'put', 'del',
+        'the', 'and', 'for', 'are', 'but', 'not', 'you', 'all', 'can', 'had', 'her',
+        'was', 'one', 'our', 'out', 'day', 'had', 'has', 'his', 'how', 'man', 'new',
+        'now', 'old', 'see', 'two', 'who', 'boy', 'did', 'its', 'let', 'put', 'say',
+        'she', 'too', 'use', 'var', 'way', 'may', 'end', 'why', 'any', 'app', 'run'
+    }
+    
     for keyword in keywords:
         if (len(keyword) >= 3 and 
-            keyword not in ['www', 'com', 'org', 'net', 'int', 'str', 'obj', 'get', 'set', 'put', 'del']):
+            keyword not in exclude_words and
+            not keyword.isdigit()):
             filtered_keywords.add(keyword)
     
     if filtered_keywords:
@@ -287,6 +365,50 @@ def _parse(line: str) -> dict | None:
     }
 
 
+def _is_malicious_context_trainer(path: str, keyword: str, status: str = "404") -> bool:
+    """
+    Determine if a keyword from log analysis appears in a malicious context.
+    This is the trainer version of the middleware's _is_malicious_context method.
+    """
+    # Don't learn from valid Django paths
+    if path_exists_in_django(path):
+        return False
+    
+    # Strong malicious indicators for log analysis
+    malicious_indicators = [
+        # Multiple suspicious segments in path
+        len([seg for seg in re.split(r"\W+", path) if seg in STATIC_KW]) > 1,
+        
+        # Common attack patterns
+        any(pattern in path.lower() for pattern in [
+            '../', '..\\', '.env', 'wp-admin', 'phpmyadmin', 'config',
+            'backup', 'database', 'mysql', 'passwd', 'shadow', 'xmlrpc',
+            'shell', 'cmd', 'exec', 'eval', 'system'
+        ]),
+        
+        # Path indicates obvious attack attempt
+        any(attack in path.lower() for attack in [
+            'union+select', 'drop+table', '<script', 'javascript:',
+            '${', '{{', 'onload=', 'onerror=', 'file://', 'http://'
+        ]),
+        
+        # Multiple directory traversal attempts
+        path.count('../') > 1 or path.count('..\\') > 1,
+        
+        # Encoded attack patterns
+        any(encoded in path for encoded in ['%2e%2e', '%252e', '%c0%ae', '%3c%73%63%72%69%70%74']),
+        
+        # 404 status with suspicious characteristics
+        status == "404" and (
+            len(path) > 50 or  # Very long paths are often attacks
+            path.count('/') > 10 or  # Too many directory levels
+            any(c in path for c in ['<', '>', '{', '}', '$', '`'])  # Special characters
+        ),
+    ]
+    
+    return any(malicious_indicators)
+
+
 def train() -> None:
     """Enhanced training with improved keyword filtering and exemption handling"""
     print("🚀 Starting AIWAF enhanced training...")
@@ -451,28 +573,43 @@ def train() -> None:
             for seg in re.split(r"\W+", r["path"].lower()):
                 if (len(seg) > 3 and 
                     seg not in STATIC_KW and 
-                    seg not in legitimate_keywords):  # Don't learn legitimate keywords
+                    seg not in legitimate_keywords and  # Don't learn legitimate keywords
+                    _is_malicious_context_trainer(r["path"], seg, r["status"])):  # Smart context check
                     tokens[seg] += 1
 
     keyword_store = get_keyword_store()
     top_tokens = tokens.most_common(getattr(settings, "AIWAF_DYNAMIC_TOP_N", 10))
     
-    # Additional filtering: only add keywords that appear suspicious enough
+    # Additional filtering: only add keywords that appear suspicious enough AND in malicious context
     filtered_tokens = []
+    learned_from_paths = []  # Track which paths we learned from
+    
     for kw, cnt in top_tokens:
-        # Don't add keywords that might be legitimate
+        # Find example paths where this keyword appeared
+        example_paths = [r["path"] for r in parsed 
+                        if kw in r["path"].lower() and 
+                        r["status"].startswith(("4", "5")) and
+                        not path_exists_in_django(r["path"])]
+        
+        # Only add if keyword appears in malicious contexts
         if (cnt >= 2 and  # Must appear at least twice
             len(kw) >= 4 and  # Must be at least 4 characters
-            kw not in legitimate_keywords):  # Not in legitimate set
+            kw not in legitimate_keywords and  # Not in legitimate set
+            example_paths and  # Has example paths
+            any(_is_malicious_context_trainer(path, kw) for path in example_paths[:3])):  # Check first 3 paths
+            
             filtered_tokens.append((kw, cnt))
             keyword_store.add_keyword(kw, cnt)
+            learned_from_paths.extend(example_paths[:2])  # Track first 2 example paths
     
     if filtered_tokens:
         print(f"📝 Added {len(filtered_tokens)} suspicious keywords: {[kw for kw, _ in filtered_tokens]}")
+        print(f"🎯 Example malicious paths learned from: {learned_from_paths[:5]}")  # Show first 5
     else:
         print("✅ No new suspicious keywords learned (good sign!)")
     
-    print(f"🎯 Dynamic keyword learning complete. Excluded {len(legitimate_keywords)} legitimate keywords.")
+    print(f"🎯 Smart keyword learning complete. Excluded {len(legitimate_keywords)} legitimate keywords.")
+    print(f"🔒 Used malicious context analysis to filter out false positives.")
     
     # Training summary
     print("\n" + "="*60)

{aiwaf-0.1.9.2.2.dist-info → aiwaf-0.1.9.2.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aiwaf
-Version: 0.1.9.2.2
+Version: 0.1.9.2.4
 Summary: AI-powered Web Application Firewall
 Home-page: https://github.com/aayushgauba/aiwaf
 Author: Aayush Gauba

{aiwaf-0.1.9.2.2.dist-info → aiwaf-0.1.9.2.4.dist-info}/RECORD

@@ -1,12 +1,12 @@
-aiwaf/__init__.py,sha256=EwzM3mRDf7i5IVX0-pTMpVFRf51lpIIbh6ZzkzOw10M,220
+aiwaf/__init__.py,sha256=dd7iB2QMRRoAnHCazKFantX-94j0ZZzF91cXNQXynwU,220
 aiwaf/apps.py,sha256=nCez-Ptlv2kaEk5HenA8b1pATz1VfhrHP1344gwcY1A,142
 aiwaf/blacklist_manager.py,sha256=LYCeKFB-7e_C6Bg2WeFJWFIIQlrfRMPuGp30ivrnhQY,1196
 aiwaf/decorators.py,sha256=IUKOdM_gdroffImRZep1g1wT6gNqD10zGwcp28hsJCs,825
-aiwaf/middleware.py,sha256=8EC4AKfUjHhmVSKpquimkMUebBekr92pqyVF97wlbx0,27408
+aiwaf/middleware.py,sha256=uUDHipAk1FBgw8B7VGRfbM8JgxQYC4rkVm_Igvwr4EU,31125
 aiwaf/middleware_logger.py,sha256=LWZVDAnjh6CGESirA8eMbhGgJKB7lVDGRQqVroH95Lo,4742
 aiwaf/models.py,sha256=vQxgY19BDVMjoO903UNrTZC1pNoLltMU6wbyWPoAEns,2719
 aiwaf/storage.py,sha256=5ImrZMRn3u7HNsPH0fDjWhDrD2tgG2IHVnOXtLz0fk4,10253
-aiwaf/trainer.py,sha256=U-X79nFhSTEbVexFHo3IXFf1HgvXrFnQ__WqTar0o4M,19118
+aiwaf/trainer.py,sha256=hswkopOnEQHNkSn79AIk-OWw6zatP2EDhsBBEHQeFYw,26410
 aiwaf/utils.py,sha256=BJk5vJCYdGPl_4QQiknjhCbkzv5HZCXgFcBJDMJpHok,3390
 aiwaf/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aiwaf/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -29,8 +29,8 @@ aiwaf/management/commands/test_exemption_fix.py,sha256=ngyGaHUCmQQ6y--6j4q1viZJt
 aiwaf/resources/model.pkl,sha256=5t6h9BX8yoh2xct85MXOO60jdlWyg1APskUOW0jZE1Y,1288265
 aiwaf/templatetags/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aiwaf/templatetags/aiwaf_tags.py,sha256=XXfb7Tl4DjU3Sc40GbqdaqOEtKTUKELBEk58u83wBNw,357
-aiwaf-0.1.9.2.2.dist-info/licenses/LICENSE,sha256=Ir8PX4dxgAcdB0wqNPIkw84fzIIRKE75NoUil9RX0QU,1069
-aiwaf-0.1.9.2.2.dist-info/METADATA,sha256=NFu9QZWsGPcmAJaHeJroSXZL_PstDr33NbSffV94bLQ,26824
-aiwaf-0.1.9.2.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-aiwaf-0.1.9.2.2.dist-info/top_level.txt,sha256=kU6EyjobT6UPCxuWpI_BvcHDG0I2tMgKaPlWzVxe2xI,6
-aiwaf-0.1.9.2.2.dist-info/RECORD,,
+aiwaf-0.1.9.2.4.dist-info/licenses/LICENSE,sha256=Ir8PX4dxgAcdB0wqNPIkw84fzIIRKE75NoUil9RX0QU,1069
+aiwaf-0.1.9.2.4.dist-info/METADATA,sha256=kxzp7zWpy5ig93vGyai9wtbcTuq5DSX7bDjNsMliTkg,26824
+aiwaf-0.1.9.2.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+aiwaf-0.1.9.2.4.dist-info/top_level.txt,sha256=kU6EyjobT6UPCxuWpI_BvcHDG0I2tMgKaPlWzVxe2xI,6
+aiwaf-0.1.9.2.4.dist-info/RECORD,,