souleyez 2.43.29__py3-none-any.whl → 3.0.0__py3-none-any.whl

souleyez/parsers/john_parser.py

@@ -12,208 +12,220 @@ from typing import Dict, List, Tuple
 def parse_john_output(output: str, hash_file: str = None) -> Dict:
     """
     Parse John the Ripper output and extract cracked passwords.
-    
+
     Args:
         output: John's stdout/stderr output
         hash_file: Path to the hash file (to run --show if needed)
-    
+
     Returns:
         Dictionary with cracked credentials
     """
     results = {
-        'cracked': [],
-        'total_loaded': 0,
-        'total_cracked': 0,
-        'session_status': 'unknown'
+        "cracked": [],
+        "total_loaded": 0,
+        "total_cracked": 0,
+        "session_status": "unknown",
     }
-    
+
     # Parse loaded hashes
-    loaded_match = re.search(r'Loaded (\d+) password hash', output)
+    loaded_match = re.search(r"Loaded (\d+) password hash", output)
     if loaded_match:
-        results['total_loaded'] = int(loaded_match.group(1))
-    
+        results["total_loaded"] = int(loaded_match.group(1))
+
     # Parse cracked passwords from live output with multiple format support
     # Format 1: "password         (username)"
     # Format 2: "password (username)"
     # Format 3: "username:password"
     # Format 4: "password          (username) [hash_type]"
-    for line in output.split('\n'):
+    for line in output.split("\n"):
         line = line.strip()
-        if not line or line.startswith('#') or line.startswith('['):
+        if not line or line.startswith("#") or line.startswith("["):
             continue
 
         # Try format: password (username) with optional hash type
-        match = re.match(r'^(\S+)\s+\(([^)]+)\)(?:\s+\[.+\])?\s*$', line)
+        match = re.match(r"^(\S+)\s+\(([^)]+)\)(?:\s+\[.+\])?\s*$", line)
         if match:
             password = match.group(1)
             username = match.group(2)
-            results['cracked'].append({
-                'username': username,
-                'password': password,
-                'source': 'john_live'
-            })
+            results["cracked"].append(
+                {"username": username, "password": password, "source": "john_live"}
+            )
             continue
 
         # Try format: username:password (from --show output)
-        if ':' in line and not line.startswith('Loaded'):
-            parts = line.split(':')
+        if ":" in line and not line.startswith("Loaded"):
+            parts = line.split(":")
             if len(parts) >= 2 and len(parts[0]) > 0 and len(parts[-1]) > 0:
                 # Skip if it looks like a hash (32+ hex chars)
-                if not re.match(r'^[0-9a-fA-F]{32,}$', parts[-1]):
+                if not re.match(r"^[0-9a-fA-F]{32,}$", parts[-1]):
                     username = parts[0]
                     password = parts[-1]
-                    results['cracked'].append({
-                        'username': username,
-                        'password': password,
-                        'source': 'john_live'
-                    })
+                    results["cracked"].append(
+                        {
+                            "username": username,
+                            "password": password,
+                            "source": "john_live",
+                        }
+                    )
 
     # Check session status with multiple format support
-    if any(x in output for x in ['Session completed', 'session completed', 'Proceeding with next']):
-        results['session_status'] = 'completed'
-    elif any(x in output for x in ['Session aborted', 'session aborted', 'Interrupted']):
-        results['session_status'] = 'aborted'
-    elif 'No password hashes left to crack' in output:
-        results['session_status'] = 'completed'
+    if any(
+        x in output
+        for x in ["Session completed", "session completed", "Proceeding with next"]
+    ):
+        results["session_status"] = "completed"
+    elif any(
+        x in output for x in ["Session aborted", "session aborted", "Interrupted"]
+    ):
+        results["session_status"] = "aborted"
+    elif "No password hashes left to crack" in output:
+        results["session_status"] = "completed"
 
     # Parse summary line with multiple formats
     # Format 1: "2g 0:00:00:01 DONE..."
     # Format 2: "2g 0:00:00:01 100% DONE..."
     # Format 3: "Session completed, 2g"
     summary_patterns = [
-        r'(\d+)g\s+[\d:]+\s+(?:\d+%\s+)?(DONE|Session)',
-        r'Session completed[,\s]+(\d+)g',
-        r'(\d+)\s+password hashes? cracked',
+        r"(\d+)g\s+[\d:]+\s+(?:\d+%\s+)?(DONE|Session)",
+        r"Session completed[,\s]+(\d+)g",
+        r"(\d+)\s+password hashes? cracked",
     ]
     for pattern in summary_patterns:
         summary_match = re.search(pattern, output, re.IGNORECASE)
         if summary_match:
-            results['total_cracked'] = int(summary_match.group(1))
+            results["total_cracked"] = int(summary_match.group(1))
             break
-    
+
     # If hash_file provided, also parse john.pot or run --show
     if hash_file and os.path.isfile(hash_file):
         pot_results = parse_john_pot(hash_file)
         # Merge with live results (pot is authoritative)
         if pot_results:
-            results['cracked'].extend(pot_results)
+            results["cracked"].extend(pot_results)
             # Deduplicate by username
             seen = set()
             unique_creds = []
-            for cred in results['cracked']:
-                if cred['username'] not in seen:
-                    seen.add(cred['username'])
+            for cred in results["cracked"]:
+                if cred["username"] not in seen:
+                    seen.add(cred["username"])
                     unique_creds.append(cred)
-            results['cracked'] = unique_creds
-            results['total_cracked'] = len(unique_creds)
-    
+            results["cracked"] = unique_creds
+            results["total_cracked"] = len(unique_creds)
+
     return results
 
 
 def parse_john_pot(hash_file: str = None) -> List[Dict]:
     """
     Parse John's potfile for cracked passwords.
-    
+
     Args:
         hash_file: If provided, run 'john --show hashfile' to get results
-    
+
     Returns:
         List of cracked credentials
     """
     cracked = []
-    
+
     # Try running john --show on the hash file
     if hash_file and os.path.isfile(hash_file):
         try:
             import subprocess
+
             result = subprocess.run(
-                ['john', '--show', hash_file],
+                ["john", "--show", hash_file],
                 capture_output=True,
                 text=True,
-                timeout=10
+                timeout=10,
             )
-            
+
             # Parse --show output
             # Format: "username:password" or "username:hash:password"
-            for line in result.stdout.split('\n'):
+            for line in result.stdout.split("\n"):
                 line = line.strip()
-                if ':' in line and not line.startswith('#'):
-                    parts = line.split(':')
+                if ":" in line and not line.startswith("#"):
+                    parts = line.split(":")
                     if len(parts) >= 2:
                         username = parts[0]
                         # Password is the last part
                         password = parts[-1]
                         if username and password:
-                            cracked.append({
-                                'username': username,
-                                'password': password,
-                                'source': 'john_pot'
-                            })
+                            cracked.append(
+                                {
+                                    "username": username,
+                                    "password": password,
+                                    "source": "john_pot",
+                                }
+                            )
         except Exception:
             pass
-    
+
     # Also try reading ~/.john/john.pot directly
-    pot_file = os.path.expanduser('~/.john/john.pot')
+    pot_file = os.path.expanduser("~/.john/john.pot")
     if os.path.isfile(pot_file):
         try:
-            with open(pot_file, 'r') as f:
+            with open(pot_file, "r") as f:
                 for line in f:
                     line = line.strip()
-                    if ':' in line:
+                    if ":" in line:
                         # Potfile format: hash:password
-                        parts = line.split(':')
+                        parts = line.split(":")
                         if len(parts) >= 2:
                             # Extract password (last part)
                             password = parts[-1]
                             # Try to find username from hash if available
                             # This is basic - john.pot doesn't always have username
-                            cracked.append({
-                                'username': 'unknown',
-                                'password': password,
-                                'hash': parts[0],
-                                'source': 'john_pot_file'
-                            })
+                            cracked.append(
+                                {
+                                    "username": "unknown",
+                                    "password": password,
+                                    "hash": parts[0],
+                                    "source": "john_pot_file",
+                                }
+                            )
         except Exception:
             pass
-    
+
     return cracked
 
 
-def extract_credentials_to_db(output: str, hash_file: str = None, engagement_id: int = None):
+def extract_credentials_to_db(
+    output: str, hash_file: str = None, engagement_id: int = None
+):
     """
     Extract cracked credentials and store them in the database.
-    
+
     Args:
         output: John's output
         hash_file: Path to hash file
         engagement_id: Current engagement ID
     """
     from souleyez.storage.credentials import CredentialManager
-    
+
     results = parse_john_output(output, hash_file)
-    
-    if not results['cracked']:
+
+    if not results["cracked"]:
         return 0
-    
+
     cred_mgr = CredentialManager()
     added = 0
-    
-    for cred in results['cracked']:
-        username = cred.get('username')
-        password = cred.get('password')
-        
-        if username and password and username != 'unknown':
+
+    for cred in results["cracked"]:
+        username = cred.get("username")
+        password = cred.get("password")
+
+        if username and password and username != "unknown":
             try:
                 # Add to credentials database
                 cred_mgr.add_credential(
                     username=username,
                     password=password,
-                    service='cracked',  # Mark as cracked hash
-                    host='',  # No specific host
-                    engagement_id=engagement_id
+                    service="cracked",  # Mark as cracked hash
+                    host="",  # No specific host
+                    engagement_id=engagement_id,
                 )
                 added += 1
             except Exception:
                 pass  # Credential might already exist
-    
+
     return added

souleyez/parsers/katana_parser.py

@@ -0,0 +1,316 @@
+#!/usr/bin/env python3
+"""
+souleyez.parsers.katana_parser - Parse Katana JSONL output
+
+Katana is a web crawler from ProjectDiscovery that discovers endpoints,
+parameters, and JavaScript-rendered routes.
+"""
+import json
+from typing import Dict, Any, List, Set
+from urllib.parse import urlparse, parse_qs
+
+# LFI-suspicious parameter names - these typically include files, not query databases
+LFI_PARAM_NAMES = {
+    # Direct file inclusion params
+    "page",
+    "file",
+    "include",
+    "inc",
+    "path",
+    "filepath",
+    "filename",
+    "template",
+    "tmpl",
+    "tpl",
+    "view",
+    "layout",
+    "content",
+    # Document/resource params
+    "doc",
+    "document",
+    "pdf",
+    "folder",
+    "root",
+    "directory",
+    "dir",
+    # Language/locale (often load language files)
+    "lang",
+    "language",
+    "locale",
+    "loc",
+    # Style/theme (often load CSS/template files)
+    "style",
+    "stylesheet",
+    "css",
+    "theme",
+    "skin",
+    # Config/module loading
+    "config",
+    "conf",
+    "cfg",
+    "module",
+    "mod",
+    "plugin",
+    # Read/load operations
+    "read",
+    "load",
+    "fetch",
+    "get",
+    "show",
+    "display",
+    "render",
+    # PHP-specific
+    "pg",
+    "p",
+    "cont",
+    "controller",
+    "action",
+    "act",
+    # Common variations
+    "pagename",
+    "page_name",
+    "pageid",
+    "site",
+    "section",
+}
+
+
+def parse_katana_output(log_path: str) -> Dict[str, Any]:
+    """
+    Parse Katana JSONL output (one JSON object per line).
+
+    Args:
+        log_path: Path to katana output file
+
+    Returns:
+        Dict containing:
+            - urls: List of all discovered URLs
+            - urls_with_params: List of URLs containing query parameters
+            - lfi_candidate_urls: URLs with LFI-suspicious params (page, file, include, etc.)
+            - sqli_candidate_urls: URLs with non-LFI params (id, q, search, etc.)
+            - forms_found: List of POST endpoint URLs
+            - js_endpoints: List of JavaScript-discovered endpoints
+            - unique_parameters: Set of unique parameter names found
+            - lfi_params_found: Set of LFI parameter names found
+            - methods: Dict of method counts (GET, POST, etc.)
+    """
+    urls: List[str] = []
+    urls_with_params: List[str] = []
+    lfi_candidate_urls: List[str] = []  # URLs with LFI-suspicious params only
+    sqli_candidate_urls: List[str] = []  # URLs with non-LFI params
+    forms_found: List[str] = []
+    js_endpoints: List[str] = []
+    unique_parameters: Set[str] = set()
+    lfi_params_found: Set[str] = set()  # Track which LFI params we found
+    methods: Dict[str, int] = {"GET": 0, "POST": 0, "PUT": 0, "DELETE": 0, "OTHER": 0}
+    status_codes: Dict[int, int] = {}
+    sources: Dict[str, int] = {}
+
+    try:
+        with open(log_path, "r", encoding="utf-8") as f:
+            for line in f:
+                # Skip comment lines and metadata
+                if line.startswith("#") or line.startswith("==="):
+                    continue
+
+                line = line.strip()
+                if not line:
+                    continue
+
+                try:
+                    result = json.loads(line)
+
+                    # Katana output format can vary by version
+                    # Try multiple field locations
+                    url = None
+                    method = "GET"
+                    status_code = None
+                    source = None
+
+                    # Format 1: request.endpoint structure
+                    if "request" in result:
+                        req = result["request"]
+                        url = req.get("endpoint") or req.get("url")
+                        method = req.get("method", "GET").upper()
+                        if "response" in result:
+                            status_code = result["response"].get("status_code")
+
+                    # Format 2: Direct fields
+                    if not url:
+                        url = result.get("endpoint") or result.get("url")
+                        method = result.get("method", "GET").upper()
+                        status_code = result.get("status_code") or result.get("status")
+
+                    # Get source (how endpoint was discovered)
+                    source = result.get("source") or result.get("tag") or "unknown"
+
+                    if not url:
+                        continue
+
+                    # Track all URLs
+                    if url not in urls:
+                        urls.append(url)
+
+                    # Track methods
+                    if method in methods:
+                        methods[method] += 1
+                    else:
+                        methods["OTHER"] += 1
+
+                    # Track status codes
+                    if status_code:
+                        status_codes[status_code] = status_codes.get(status_code, 0) + 1
+
+                    # Track sources
+                    sources[source] = sources.get(source, 0) + 1
+
+                    # Check for query parameters
+                    parsed = urlparse(url)
+                    if parsed.query:
+                        if url not in urls_with_params:
+                            urls_with_params.append(url)
+
+                        # Extract parameter names and categorize
+                        # Use keep_blank_values=True to detect params like ?q= (empty value)
+                        params = parse_qs(parsed.query, keep_blank_values=True)
+                        param_names = set(params.keys())
+                        unique_parameters.update(param_names)
+
+                        # Check if params are LFI-suspicious
+                        lfi_params_in_url = param_names & LFI_PARAM_NAMES
+                        non_lfi_params = param_names - LFI_PARAM_NAMES
+
+                        if lfi_params_in_url:
+                            lfi_params_found.update(lfi_params_in_url)
+
+                        # Categorize URL based on params
+                        if lfi_params_in_url and not non_lfi_params:
+                            # ALL params are LFI-suspicious → LFI candidate only
+                            if url not in lfi_candidate_urls:
+                                lfi_candidate_urls.append(url)
+                        elif non_lfi_params:
+                            # Has non-LFI params → SQLi candidate
+                            if url not in sqli_candidate_urls:
+                                sqli_candidate_urls.append(url)
+                            # Also add to LFI if it has LFI params (mixed)
+                            if lfi_params_in_url and url not in lfi_candidate_urls:
+                                lfi_candidate_urls.append(url)
+
+                    # Track POST endpoints as forms
+                    if method == "POST":
+                        if url not in forms_found:
+                            forms_found.append(url)
+
+                    # Track JavaScript-discovered endpoints
+                    if source in ("js", "script", "javascript", "jscrawl"):
+                        if url not in js_endpoints:
+                            js_endpoints.append(url)
+
+                except json.JSONDecodeError:
+                    # Skip non-JSON lines (like metadata headers)
+                    continue
+
+    except FileNotFoundError:
+        return {
+            "urls": [],
+            "urls_with_params": [],
+            "lfi_candidate_urls": [],
+            "sqli_candidate_urls": [],
+            "forms_found": [],
+            "js_endpoints": [],
+            "unique_parameters": [],
+            "lfi_params_found": [],
+            "methods": methods,
+            "status_codes": {},
+            "sources": {},
+            "error": f"Log file not found: {log_path}",
+        }
+    except Exception as e:
+        return {
+            "urls": [],
+            "urls_with_params": [],
+            "lfi_candidate_urls": [],
+            "sqli_candidate_urls": [],
+            "forms_found": [],
+            "js_endpoints": [],
+            "unique_parameters": [],
+            "lfi_params_found": [],
+            "methods": methods,
+            "status_codes": {},
+            "sources": {},
+            "error": str(e),
+        }
+
+    return {
+        "urls": urls,
+        "urls_with_params": urls_with_params,
+        "lfi_candidate_urls": lfi_candidate_urls,
+        "sqli_candidate_urls": sqli_candidate_urls,
+        "forms_found": forms_found,
+        "js_endpoints": js_endpoints,
+        "unique_parameters": sorted(list(unique_parameters)),
+        "lfi_params_found": sorted(list(lfi_params_found)),
+        "methods": methods,
+        "status_codes": status_codes,
+        "sources": sources,
+    }
+
+
+def extract_injectable_urls(parsed_data: Dict[str, Any]) -> List[str]:
+    """
+    Extract URLs that are good candidates for SQL injection testing.
+
+    Only returns URLs with non-LFI parameters. URLs with only LFI-suspicious
+    params (page, file, include, etc.) are excluded since SQLMap won't find
+    LFI vulnerabilities.
+
+    Prioritizes:
+    1. URLs with non-LFI query parameters (sqli_candidate_urls)
+    2. POST form endpoints
+    3. JavaScript-discovered API endpoints
+
+    Args:
+        parsed_data: Output from parse_katana_output()
+
+    Returns:
+        List of URLs suitable for SQLMap/SQL injection testing
+    """
+    injectable = []
+
+    # SQLi candidate URLs (have non-LFI params)
+    for url in parsed_data.get("sqli_candidate_urls", []):
+        if url not in injectable:
+            injectable.append(url)
+
+    # POST forms are also injectable, but skip LFI-only forms
+    lfi_candidates = parsed_data.get("lfi_candidate_urls", [])
+    sqli_candidates = parsed_data.get("sqli_candidate_urls", [])
+    for url in parsed_data.get("forms_found", []):
+        if url not in injectable:
+            # Skip forms that only have LFI params (no SQLi potential)
+            if url in lfi_candidates and url not in sqli_candidates:
+                continue
+            injectable.append(url)
+
+    # JS endpoints might have hidden params
+    for url in parsed_data.get("js_endpoints", []):
+        if url not in injectable:
+            injectable.append(url)
+
+    return injectable
+
+
+def extract_lfi_urls(parsed_data: Dict[str, Any]) -> List[str]:
+    """
+    Extract URLs that are good candidates for LFI (Local File Inclusion) testing.
+
+    Returns URLs with LFI-suspicious parameters like page, file, include, path,
+    template, etc. These should be tested with LFI payloads, not SQLMap.
+
+    Args:
+        parsed_data: Output from parse_katana_output()
+
+    Returns:
+        List of URLs suitable for LFI testing
+    """
+    return parsed_data.get("lfi_candidate_urls", [])