souleyez 2.43.26__py3-none-any.whl → 2.43.34__py3-none-any.whl

souleyez/parsers/theharvester_parser.py

@@ -47,102 +47,120 @@ def parse_theharvester_output(output: str, target: str = "") -> Dict[str, Any]:
         }
     """
     result = {
-        'target': target,
-        'emails': [],
-        'hosts': [],
-        'ips': [],
-        'urls': [],
-        'asns': []
+        "target": target,
+        "emails": [],
+        "hosts": [],
+        "ips": [],
+        "urls": [],
+        "asns": [],
     }
 
-    lines = output.split('\n')
+    lines = output.split("\n")
     current_section = None
 
     for line in lines:
         line = line.strip()
 
         # Detect target
-        if line.startswith('[*] Target:'):
-            target_match = re.search(r'\[?\*\]?\s*Target:\s*(\S+)', line)
+        if line.startswith("[*] Target:"):
+            target_match = re.search(r"\[?\*\]?\s*Target:\s*(\S+)", line)
             if target_match:
-                result['target'] = target_match.group(1)
+                result["target"] = target_match.group(1)
 
         # Detect section headers (case-insensitive, multiple format variations)
         line_lower = line.lower()
-        if any(x in line_lower for x in ['asns found', 'asn found', 'autonomous system']):
-            current_section = 'asns'
-        elif any(x in line_lower for x in ['urls found', 'interesting urls', 'url found']):
-            current_section = 'urls'
-        elif any(x in line_lower for x in ['ips found', 'ip found', 'ip addresses']):
-            current_section = 'ips'
-        elif any(x in line_lower for x in ['emails found', 'email found', 'email addresses']):
-            current_section = 'emails'
-        elif any(x in line_lower for x in ['hosts found', 'host found', 'subdomains found', 'subdomain found']):
-            current_section = 'hosts'
-        elif any(x in line_lower for x in ['people found', 'no people found', 'linkedin']):
-            current_section = 'people'  # We'll skip this for now
+        if any(
+            x in line_lower for x in ["asns found", "asn found", "autonomous system"]
+        ):
+            current_section = "asns"
+        elif any(
+            x in line_lower for x in ["urls found", "interesting urls", "url found"]
+        ):
+            current_section = "urls"
+        elif any(x in line_lower for x in ["ips found", "ip found", "ip addresses"]):
+            current_section = "ips"
+        elif any(
+            x in line_lower for x in ["emails found", "email found", "email addresses"]
+        ):
+            current_section = "emails"
+        elif any(
+            x in line_lower
+            for x in [
+                "hosts found",
+                "host found",
+                "subdomains found",
+                "subdomain found",
+            ]
+        ):
+            current_section = "hosts"
+        elif any(
+            x in line_lower for x in ["people found", "no people found", "linkedin"]
+        ):
+            current_section = "people"  # We'll skip this for now
 
         # Skip separator lines and empty lines
-        elif line.startswith('---') or not line:
+        elif line.startswith("---") or not line:
             continue
 
         # Skip "No X found" messages
-        elif '[*] No' in line:
+        elif "[*] No" in line:
             current_section = None
             continue
 
         # Skip header/banner lines
-        elif line.startswith('*') or line.startswith('[*] Searching'):
+        elif line.startswith("*") or line.startswith("[*] Searching"):
             continue
 
         # Parse data based on current section
-        elif current_section == 'asns':
+        elif current_section == "asns":
             # ASN format: AS12345
-            if line.startswith('AS') and line[2:].isdigit():
-                result['asns'].append(line)
+            if line.startswith("AS") and line[2:].isdigit():
+                result["asns"].append(line)
 
-        elif current_section == 'urls':
+        elif current_section == "urls":
             # URL format: http(s)://...
-            if line.startswith('http://') or line.startswith('https://'):
+            if line.startswith("http://") or line.startswith("https://"):
                 # Clean up trailing punctuation
-                url = line.rstrip('.,;)')
-                if url not in result['urls']:
-                    result['urls'].append(url)
+                url = line.rstrip(".,;)")
+                if url not in result["urls"]:
+                    result["urls"].append(url)
 
-        elif current_section == 'ips':
+        elif current_section == "ips":
             # IP format: N.N.N.N
-            if re.match(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$', line):
-                if line not in result['ips']:
-                    result['ips'].append(line)
+            if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", line):
+                if line not in result["ips"]:
+                    result["ips"].append(line)
 
-        elif current_section == 'emails':
+        elif current_section == "emails":
             # Email format: user@domain
-            if '@' in line and '.' in line:
+            if "@" in line and "." in line:
                 # More permissive email validation (supports international domains)
                 # Pattern allows: standard emails, plus-addressing, dots, underscores
                 email = line.strip().lower()
                 # Remove any leading/trailing brackets or quotes
-                email = re.sub(r'^[\[\(<\'\"]+|[\]\)>\'\"]$', '', email)
-                if re.match(r'^[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}$', email):
-                    if email not in result['emails']:
-                        result['emails'].append(email)
-
-        elif current_section == 'hosts':
+                email = re.sub(r"^[\[\(<\'\"]+|[\]\)>\'\"]$", "", email)
+                if re.match(
+                    r"^[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}$", email
+                ):
+                    if email not in result["emails"]:
+                        result["emails"].append(email)
+
+        elif current_section == "hosts":
             # Host format: subdomain.domain.tld
-            if '.' in line and not line.startswith('http'):
+            if "." in line and not line.startswith("http"):
                 # Clean and validate hostname
                 host = line.strip().lower()
                 # Remove any leading/trailing brackets, quotes, or trailing dots
-                host = re.sub(r'^[\[\(<\'\"]+|[\]\)>\'\".]+$', '', host)
+                host = re.sub(r"^[\[\(<\'\"]+|[\]\)>\'\".]+$", "", host)
                 # More permissive validation: allows underscores (common in some hosts)
                 # and longer TLDs (some are 4+ chars)
-                if re.match(r'^[a-zA-Z0-9._-]+\.[a-zA-Z]{2,}$', host) and len(host) > 3:
-                    if host not in result['hosts']:
-                        result['hosts'].append(host)
+                if re.match(r"^[a-zA-Z0-9._-]+\.[a-zA-Z]{2,}$", host) and len(host) > 3:
+                    if host not in result["hosts"]:
+                        result["hosts"].append(host)
 
     # Add alias fields for backward compatibility with display code
-    result['subdomains'] = result['hosts']  # Alias for display
-    result['base_urls'] = result['urls']     # Alias for display
+    result["subdomains"] = result["hosts"]  # Alias for display
+    result["base_urls"] = result["urls"]  # Alias for display
 
     return result
 
@@ -158,9 +176,9 @@ def get_osint_stats(parsed: Dict[str, Any]) -> Dict[str, int]:
         Dict with counts: {'emails': 5, 'hosts': 10, ...}
     """
     return {
-        'emails': len(parsed.get('emails', [])),
-        'hosts': len(parsed.get('hosts', [])),
-        'ips': len(parsed.get('ips', [])),
-        'urls': len(parsed.get('urls', [])),
-        'asns': len(parsed.get('asns', []))
+        "emails": len(parsed.get("emails", [])),
+        "hosts": len(parsed.get("hosts", [])),
+        "ips": len(parsed.get("ips", [])),
+        "urls": len(parsed.get("urls", [])),
+        "asns": len(parsed.get("asns", [])),
     }

souleyez/parsers/whois_parser.py

@@ -51,123 +51,149 @@ def parse_whois_output(output: str, target: str = "") -> Dict[str, Any]:
         }
     """
     result = {
-        'domain': target,
-        'registrar': None,
-        'registrant': {},
-        'admin_contact': {},
-        'tech_contact': {},
-        'dates': {},
-        'nameservers': [],
-        'status': [],
-        'dnssec': None
+        "domain": target,
+        "registrar": None,
+        "registrant": {},
+        "admin_contact": {},
+        "tech_contact": {},
+        "dates": {},
+        "nameservers": [],
+        "status": [],
+        "dnssec": None,
     }
 
-    lines = output.split('\n')
+    lines = output.split("\n")
     current_section = None
 
     for line in lines:
         line_stripped = line.strip()
 
         # Skip comments and empty lines
-        if not line_stripped or line_stripped.startswith('%') or line_stripped.startswith('#'):
+        if (
+            not line_stripped
+            or line_stripped.startswith("%")
+            or line_stripped.startswith("#")
+        ):
             continue
 
         # Convert to lowercase for matching
         line_lower = line_stripped.lower()
 
         # Extract domain name
-        if not result['domain'] or result['domain'] == '':
-            domain_match = re.match(r'domain name:\s+(.+)', line_stripped, re.IGNORECASE)
+        if not result["domain"] or result["domain"] == "":
+            domain_match = re.match(
+                r"domain name:\s+(.+)", line_stripped, re.IGNORECASE
+            )
             if domain_match:
-                result['domain'] = domain_match.group(1).strip()
+                result["domain"] = domain_match.group(1).strip()
 
         # Extract registrar
-        if 'registrar:' in line_lower and not result['registrar']:
-            registrar_match = re.search(r'registrar:\s+(.+)', line_stripped, re.IGNORECASE)
+        if "registrar:" in line_lower and not result["registrar"]:
+            registrar_match = re.search(
+                r"registrar:\s+(.+)", line_stripped, re.IGNORECASE
+            )
             if registrar_match:
-                result['registrar'] = registrar_match.group(1).strip()
+                result["registrar"] = registrar_match.group(1).strip()
 
         # Extract dates
-        if 'creation date' in line_lower or 'created' in line_lower:
-            date_match = re.search(r':\s+(\d{4}-\d{2}-\d{2}|\d{2}/\d{2}/\d{4})', line_stripped)
+        if "creation date" in line_lower or "created" in line_lower:
+            date_match = re.search(
+                r":\s+(\d{4}-\d{2}-\d{2}|\d{2}/\d{2}/\d{4})", line_stripped
+            )
             if date_match:
-                result['dates']['created'] = date_match.group(1).strip()
-
-        if 'updated date' in line_lower or 'last updated' in line_lower or 'modified' in line_lower:
-            date_match = re.search(r':\s+(\d{4}-\d{2}-\d{2}|\d{2}/\d{2}/\d{4})', line_stripped)
+                result["dates"]["created"] = date_match.group(1).strip()
+
+        if (
+            "updated date" in line_lower
+            or "last updated" in line_lower
+            or "modified" in line_lower
+        ):
+            date_match = re.search(
+                r":\s+(\d{4}-\d{2}-\d{2}|\d{2}/\d{2}/\d{4})", line_stripped
+            )
             if date_match:
-                result['dates']['updated'] = date_match.group(1).strip()
+                result["dates"]["updated"] = date_match.group(1).strip()
 
-        if 'expir' in line_lower:
-            date_match = re.search(r':\s+(\d{4}-\d{2}-\d{2}|\d{2}/\d{2}/\d{4})', line_stripped)
+        if "expir" in line_lower:
+            date_match = re.search(
+                r":\s+(\d{4}-\d{2}-\d{2}|\d{2}/\d{2}/\d{4})", line_stripped
+            )
             if date_match:
-                result['dates']['expires'] = date_match.group(1).strip()
+                result["dates"]["expires"] = date_match.group(1).strip()
 
         # Detect contact sections
-        if 'registrant' in line_lower and 'name:' in line_lower:
-            current_section = 'registrant'
-        elif 'admin' in line_lower and ('name:' in line_lower or 'contact' in line_lower):
-            current_section = 'admin'
-        elif 'tech' in line_lower and ('name:' in line_lower or 'contact' in line_lower):
-            current_section = 'tech'
+        if "registrant" in line_lower and "name:" in line_lower:
+            current_section = "registrant"
+        elif "admin" in line_lower and (
+            "name:" in line_lower or "contact" in line_lower
+        ):
+            current_section = "admin"
+        elif "tech" in line_lower and (
+            "name:" in line_lower or "contact" in line_lower
+        ):
+            current_section = "tech"
 
         # Extract contact information based on current section
         if current_section:
             contact_dict = _get_contact_dict(result, current_section)
 
-            if 'name:' in line_lower and 'domain name' not in line_lower:
-                name_match = re.search(r'name:\s+(.+)', line_stripped, re.IGNORECASE)
+            if "name:" in line_lower and "domain name" not in line_lower:
+                name_match = re.search(r"name:\s+(.+)", line_stripped, re.IGNORECASE)
                 if name_match:
-                    contact_dict['name'] = name_match.group(1).strip()
+                    contact_dict["name"] = name_match.group(1).strip()
 
-            if 'organi' in line_lower:
-                org_match = re.search(r'organi[zs]ation:\s+(.+)', line_stripped, re.IGNORECASE)
+            if "organi" in line_lower:
+                org_match = re.search(
+                    r"organi[zs]ation:\s+(.+)", line_stripped, re.IGNORECASE
+                )
                 if org_match:
-                    contact_dict['organization'] = org_match.group(1).strip()
+                    contact_dict["organization"] = org_match.group(1).strip()
 
-            if 'email' in line_lower:
-                email_match = re.search(r'email:\s+(.+)', line_stripped, re.IGNORECASE)
+            if "email" in line_lower:
+                email_match = re.search(r"email:\s+(.+)", line_stripped, re.IGNORECASE)
                 if email_match:
-                    contact_dict['email'] = email_match.group(1).strip()
+                    contact_dict["email"] = email_match.group(1).strip()
 
-            if 'phone' in line_lower:
-                phone_match = re.search(r'phone:\s+(.+)', line_stripped, re.IGNORECASE)
+            if "phone" in line_lower:
+                phone_match = re.search(r"phone:\s+(.+)", line_stripped, re.IGNORECASE)
                 if phone_match:
-                    contact_dict['phone'] = phone_match.group(1).strip()
+                    contact_dict["phone"] = phone_match.group(1).strip()
 
         # Extract nameservers
-        if 'name server' in line_lower or 'nserver' in line_lower:
-            ns_match = re.search(r'(?:name server|nserver):\s+(.+)', line_stripped, re.IGNORECASE)
+        if "name server" in line_lower or "nserver" in line_lower:
+            ns_match = re.search(
+                r"(?:name server|nserver):\s+(.+)", line_stripped, re.IGNORECASE
+            )
             if ns_match:
                 nameserver = ns_match.group(1).strip().lower()
-                if nameserver not in result['nameservers']:
-                    result['nameservers'].append(nameserver)
+                if nameserver not in result["nameservers"]:
+                    result["nameservers"].append(nameserver)
 
         # Extract status
-        if 'status:' in line_lower:
-            status_match = re.search(r'status:\s+(.+)', line_stripped, re.IGNORECASE)
+        if "status:" in line_lower:
+            status_match = re.search(r"status:\s+(.+)", line_stripped, re.IGNORECASE)
             if status_match:
                 status = status_match.group(1).strip()
-                if status not in result['status']:
-                    result['status'].append(status)
+                if status not in result["status"]:
+                    result["status"].append(status)
 
         # Extract DNSSEC
-        if 'dnssec:' in line_lower:
-            dnssec_match = re.search(r'dnssec:\s+(.+)', line_stripped, re.IGNORECASE)
+        if "dnssec:" in line_lower:
+            dnssec_match = re.search(r"dnssec:\s+(.+)", line_stripped, re.IGNORECASE)
             if dnssec_match:
-                result['dnssec'] = dnssec_match.group(1).strip()
+                result["dnssec"] = dnssec_match.group(1).strip()
 
     return result
 
 
 def _get_contact_dict(result: Dict[str, Any], section: str) -> Dict[str, Any]:
     """Get the appropriate contact dictionary based on section."""
-    if section == 'registrant':
-        return result['registrant']
-    elif section == 'admin':
-        return result['admin_contact']
-    elif section == 'tech':
-        return result['tech_contact']
+    if section == "registrant":
+        return result["registrant"]
+    elif section == "admin":
+        return result["admin_contact"]
+    elif section == "tech":
+        return result["tech_contact"]
     return {}
 
 
@@ -183,17 +209,21 @@ def extract_emails(parsed_data: Dict[str, Any]) -> List[str]:
     """
     emails = []
 
-    for contact in [parsed_data.get('registrant', {}),
-                    parsed_data.get('admin_contact', {}),
-                    parsed_data.get('tech_contact', {})]:
-        email = contact.get('email')
-        if email and email not in emails and '@' in email:
+    for contact in [
+        parsed_data.get("registrant", {}),
+        parsed_data.get("admin_contact", {}),
+        parsed_data.get("tech_contact", {}),
+    ]:
+        email = contact.get("email")
+        if email and email not in emails and "@" in email:
             emails.append(email)
 
     return emails
 
 
-def map_to_osint_data(parsed_data: Dict[str, Any], engagement_id: int) -> Dict[str, Any]:
+def map_to_osint_data(
+    parsed_data: Dict[str, Any], engagement_id: int
+) -> Dict[str, Any]:
     """
     Convert parsed WHOIS data into OSINT record for database storage.
 
@@ -209,36 +239,42 @@ def map_to_osint_data(parsed_data: Dict[str, Any], engagement_id: int) -> Dict[s
     # Extract key information for quick reference
     summary_parts = []
 
-    if parsed_data.get('registrar'):
+    if parsed_data.get("registrar"):
         summary_parts.append(f"Registrar: {parsed_data['registrar']}")
 
-    if parsed_data.get('dates', {}).get('created'):
+    if parsed_data.get("dates", {}).get("created"):
         summary_parts.append(f"Created: {parsed_data['dates']['created']}")
 
-    if parsed_data.get('dates', {}).get('expires'):
+    if parsed_data.get("dates", {}).get("expires"):
         summary_parts.append(f"Expires: {parsed_data['dates']['expires']}")
 
-    if parsed_data.get('registrant', {}).get('organization'):
+    if parsed_data.get("registrant", {}).get("organization"):
         summary_parts.append(f"Org: {parsed_data['registrant']['organization']}")
 
-    summary = " | ".join(summary_parts) if summary_parts else "Domain registration information"
+    summary = (
+        " | ".join(summary_parts)
+        if summary_parts
+        else "Domain registration information"
+    )
 
     # Extract emails for OSINT correlation
     emails = extract_emails(parsed_data)
 
     osint_record = {
-        'target': parsed_data.get('domain', ''),
-        'data_type': 'domain_info',
-        'source': 'whois',
-        'content': json.dumps(parsed_data, indent=2),
-        'summary': summary,
-        'metadata': json.dumps({
-            'registrar': parsed_data.get('registrar'),
-            'nameservers': parsed_data.get('nameservers', []),
-            'emails': emails,
-            'expiration_date': parsed_data.get('dates', {}).get('expires'),
-            'dnssec': parsed_data.get('dnssec')
-        })
+        "target": parsed_data.get("domain", ""),
+        "data_type": "domain_info",
+        "source": "whois",
+        "content": json.dumps(parsed_data, indent=2),
+        "summary": summary,
+        "metadata": json.dumps(
+            {
+                "registrar": parsed_data.get("registrar"),
+                "nameservers": parsed_data.get("nameservers", []),
+                "emails": emails,
+                "expiration_date": parsed_data.get("dates", {}).get("expires"),
+                "dnssec": parsed_data.get("dnssec"),
+            }
+        ),
     }
 
     return osint_record
@@ -255,23 +291,23 @@ def check_privacy_protection(parsed_data: Dict[str, Any]) -> bool:
         True if privacy protection detected
     """
     privacy_indicators = [
-        'privacy',
-        'redacted',
-        'protected',
-        'whoisguard',
-        'domains by proxy',
-        'private registration'
+        "privacy",
+        "redacted",
+        "protected",
+        "whoisguard",
+        "domains by proxy",
+        "private registration",
     ]
 
     # Check registrant info
-    registrant_text = str(parsed_data.get('registrant', {})).lower()
+    registrant_text = str(parsed_data.get("registrant", {})).lower()
 
     for indicator in privacy_indicators:
         if indicator in registrant_text:
             return True
 
     # Check registrar
-    registrar = str(parsed_data.get('registrar', '')).lower()
+    registrar = str(parsed_data.get("registrar", "")).lower()
     for indicator in privacy_indicators:
         if indicator in registrar:
             return True
@@ -280,4 +316,9 @@ def check_privacy_protection(parsed_data: Dict[str, Any]) -> bool:
 
 
 # Export the main functions
-__all__ = ['parse_whois_output', 'map_to_osint_data', 'extract_emails', 'check_privacy_protection']
+__all__ = [
+    "parse_whois_output",
+    "map_to_osint_data",
+    "extract_emails",
+    "check_privacy_protection",
+]