leadintel 0.3.0__py3-none-any.whl

leadintel/services/url_validator.py

@@ -0,0 +1,295 @@
+"""
+URL and Host Validation Service for SSRF protection and security checks.
+
+Prevents SSRF and DNS Rebinding attacks by:
+- Enforcing http/https schemes only.
+- Rejecting userinfo/credentials in URLs.
+- Rejecting dangerous service ports (e.g., SSH, SMTP, Redis, DB ports).
+- Blocking localhost, 127.0.0.1, ::1, 0.0.0.0, broadcasthost, and internal hostnames.
+- Blocking non-global IP addresses, link-local, loopbacks, and private subnets (RFC 1918, RFC 6598 CGNAT).
+- Blocking cloud metadata endpoints (169.254.169.254, metadata.google.internal, fd00:ec2::254, 100.100.100.200).
+- Blocking internal and local domain suffixes (.local, .localhost, .internal, .lan, .corp, .home).
+- Mitigating DNS Rebinding & TOCTOU: Validating all mixed DNS answers and pinning the TCP socket connection directly to the verified public IP.
+- Bypassing ambient proxies (trust_env=False) to prevent proxy-based SSRF bypass.
+- Validating HTTP redirects to prevent open redirect SSRF bypasses.
+- Providing safe pre-configured HTTPX clients and transports.
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import logging
+import socket
+from typing import Any, Optional
+from urllib.parse import urljoin, urlparse
+
+import httpcore
+import httpx
+from httpcore._backends.anyio import AnyIOBackend, AnyIOStream
+
+logger = logging.getLogger(__name__)
+
+# Cloud metadata IPs, loopbacks, and internal hostnames explicitly blocked
+BLOCKED_IP_STRINGS = {
+    "169.254.169.254",          # AWS/GCP/Azure/DigitalOcean metadata (IPv4)
+    "fd00:ec2::254",            # AWS IMDSv2 (IPv6)
+    "100.100.100.200",          # Alibaba Cloud metadata
+    "metadata.google.internal", # GCP metadata DNS
+    "metadata.google",          # GCP metadata alias
+    "metadata",                 # Cloud generic metadata hostname
+    "instance-data",            # EC2 instance data hostname
+}
+
+BLOCKED_HOSTNAMES = {
+    "localhost",
+    "localhost.localdomain",
+    "127.0.0.1",
+    "::1",
+    "0.0.0.0",
+    "broadcasthost",
+}
+
+BLOCKED_HOST_SUFFIXES = (
+    ".local",
+    ".localhost",
+    ".internal",
+    ".lan",
+    ".localdomain",
+    ".corp",
+    ".home",
+)
+
+# Ports associated with sensitive backend services that should never be probed by a website crawler
+DANGEROUS_PORTS = {
+    21,    # FTP
+    22,    # SSH
+    23,    # Telnet
+    25,    # SMTP
+    53,    # DNS
+    110,   # POP3
+    111,   # RPC
+    135,   # RPC
+    139,   # NetBIOS
+    143,   # IMAP
+    445,   # SMB
+    1433,  # MSSQL
+    1521,  # Oracle
+    2049,  # NFS
+    3306,  # MySQL
+    3389,  # RDP
+    5432,  # PostgreSQL
+    5900,  # VNC
+    6379,  # Redis
+    11211, # Memcached
+    27017, # MongoDB
+    28017, # MongoDB Web
+}
+
+
+class SecurityValidationError(ValueError):
+    """Raised when a URL violates safety or SSRF restrictions."""
+
+
+def is_ip_private_or_restricted(ip_str: str) -> bool:
+    """
+    Check if an IP string belongs to private, loopback, link-local, multicast,
+    non-global, or reserved ranges.
+    """
+    try:
+        ip = ipaddress.ip_address(ip_str.strip("[]"))
+        # Check IPv4-mapped IPv6 address (e.g. ::ffff:127.0.0.1)
+        mapped = getattr(ip, "ipv4_mapped", None)
+        if mapped and (not mapped.is_global or mapped.is_private or mapped.is_loopback):
+            return True
+
+        return (
+            not ip.is_global
+            or ip.is_private
+            or ip.is_loopback
+            or ip.is_link_local
+            or ip.is_multicast
+            or ip.is_reserved
+            or ip.is_unspecified
+        )
+    except ValueError:
+        return True
+
+
+def validate_and_normalize_url(url: str) -> str:
+    """
+    Validate a URL string for safety against SSRF and syntax issues.
+
+    Returns the normalized URL string if valid.
+    Raises SecurityValidationError or ValueError if invalid or unsafe.
+    """
+    if not url or not isinstance(url, str):
+        raise ValueError("URL must be a non-empty string.")
+
+    url = url.strip()
+    parsed_initial = urlparse(url)
+    if parsed_initial.scheme and parsed_initial.scheme not in ("http", "https"):
+        raise SecurityValidationError(f"Invalid scheme '{parsed_initial.scheme}'. Only http and https are allowed.")
+
+    if not (url.startswith("http://") or url.startswith("https://")):
+        url = "https://" + url
+
+    parsed = urlparse(url)
+
+    if parsed.scheme not in ("http", "https"):
+        raise SecurityValidationError(f"Invalid scheme '{parsed.scheme}'. Only http and https are allowed.")
+
+    # Disallow credentials in URLs to prevent credential confusion/leakage
+    if parsed.username or parsed.password:
+        raise SecurityValidationError("URLs containing embedded username or password credentials are not permitted.")
+
+    hostname = parsed.hostname
+    if not hostname:
+        raise SecurityValidationError("URL missing valid hostname.")
+
+    hostname_lower = hostname.lower().strip("[]")
+
+    if hostname_lower in BLOCKED_HOSTNAMES or hostname_lower in BLOCKED_IP_STRINGS:
+        raise SecurityValidationError(f"Access to restricted host '{hostname}' is blocked for security reasons (SSRF protection).")
+
+    if hostname_lower.endswith(BLOCKED_HOST_SUFFIXES):
+        raise SecurityValidationError(f"Access to internal domain '{hostname}' is blocked for security reasons (SSRF protection).")
+
+    # Check dangerous port restrictions
+    if parsed.port and parsed.port in DANGEROUS_PORTS:
+        raise SecurityValidationError(f"Access to dangerous port {parsed.port} is blocked for security reasons.")
+
+    # Check if hostname directly represents an IP literal
+    try:
+        if is_ip_private_or_restricted(hostname_lower):
+            raise SecurityValidationError(f"Access to IP '{hostname}' is restricted (private/loopback/cloud metadata).")
+    except ValueError:
+        pass
+
+    # Resolve hostname via DNS to verify target IP isn't private / non-global.
+    # Checks ALL returned addresses (handles mixed IPv4/IPv6 answers).
+    try:
+        addrinfo = socket.getaddrinfo(hostname, None)
+        for family, socktype, proto, canonname, sockaddr in addrinfo:
+            ip_str = sockaddr[0]
+            if is_ip_private_or_restricted(ip_str) or ip_str in BLOCKED_IP_STRINGS:
+                raise SecurityValidationError(
+                    f"Host '{hostname}' resolved to restricted IP '{ip_str}' (SSRF protection)."
+                )
+    except socket.gaierror as e:
+        logger.debug("DNS resolution failed for '%s': %s", hostname, e)
+        # We allow DNS resolution failures to proceed to HTTP request where HTTPX will cleanly handle connection error,
+        # unless it was specifically blocked above.
+
+    return url
+
+
+async def validate_redirect_hook(response: httpx.Response) -> None:
+    """
+    HTTPX event hook that intercepts 3xx redirects and validates the destination URL against SSRF rules.
+    Prevents malicious open redirects from redirecting crawler requests to private IPs or cloud metadata.
+    """
+    if response.is_redirect:
+        location = response.headers.get("Location")
+        if location:
+            target_url = urljoin(str(response.url), location)
+            # Validates scheme, port, IP address, hostname, and restricts private IPs/metadata
+            validate_and_normalize_url(target_url)
+
+
+class RebindingSafeAnyIOBackend(AnyIOBackend):
+    """
+    AnyIO Network backend that validates DNS at connection time and pins the TCP socket
+    directly to the verified public IP. Eliminates DNS Rebinding TOCTOU vulnerabilities
+    and detects mixed IPv4/IPv6 DNS answers containing restricted IPs.
+    """
+
+    async def connect_tcp(
+        self,
+        host: str,
+        port: int,
+        timeout: Optional[float] = None,
+        local_address: Optional[str] = None,
+        socket_options: Optional[Any] = None,
+    ) -> AnyIOStream:
+        host_clean = host.strip("[]")
+        # Check blocked hostnames and suffixes directly
+        if host_clean.lower() in BLOCKED_HOSTNAMES or host_clean.lower() in BLOCKED_IP_STRINGS:
+            raise SecurityValidationError(f"Access to restricted host '{host}' blocked at connection time.")
+
+        if host_clean.lower().endswith(BLOCKED_HOST_SUFFIXES):
+            raise SecurityValidationError(f"Access to internal domain '{host}' blocked at connection time.")
+
+        # Resolve host at the exact connection instant
+        addrinfo = socket.getaddrinfo(host_clean, port, proto=socket.IPPROTO_TCP)
+        safe_ip: Optional[str] = None
+
+        # Verify EVERY IP in the answer set (detect mixed DNS answers)
+        for family, socktype, proto, canonname, sockaddr in addrinfo:
+            ip_str = sockaddr[0]
+            if is_ip_private_or_restricted(ip_str) or ip_str in BLOCKED_IP_STRINGS:
+                raise SecurityValidationError(
+                    f"DNS Rebinding / SSRF blocked: {host} resolved to restricted IP '{ip_str}' at connection time."
+                )
+            if safe_ip is None:
+                safe_ip = ip_str
+
+        target_connect_ip = safe_ip if safe_ip else host_clean
+
+        # Connect directly to the validated IP to pin the TCP socket
+        return await super().connect_tcp(
+            target_connect_ip,
+            port,
+            timeout=timeout,
+            local_address=local_address,
+            socket_options=socket_options,
+        )
+
+
+class SafeAsyncHTTPTransport(httpx.AsyncHTTPTransport):
+    """
+    HTTPX transport pre-configured with DNS rebinding defense, connection pinning,
+    and ambient proxy bypass (trust_env=False).
+    """
+
+    def __init__(self, *args, **kwargs):
+        kwargs["trust_env"] = False  # Bypasses ambient HTTP_PROXY/ALL_PROXY variables
+        super().__init__(*args, **kwargs)
+        # Configure connection pool with rebinding-safe network backend
+        self._pool = httpcore.AsyncConnectionPool(
+            ssl_context=self._pool._ssl_context,
+            network_backend=RebindingSafeAnyIOBackend(),
+            http1=True,
+            http2=False,
+        )
+
+
+def create_safe_client(
+    timeout: float = 15.0,
+    max_redirects: int = 5,
+    headers: Optional[dict[str, str]] = None,
+) -> httpx.AsyncClient:
+    """
+    Create an httpx.AsyncClient pre-configured with:
+    - DNS rebinding mitigation (TCP connection pinning to validated IP)
+    - SSRF redirect validation hook
+    - Ambient proxy bypass (trust_env=False)
+    - Strict timeout and redirect limits
+    - Respectful User-Agent
+    """
+    client_headers = {
+        "User-Agent": "LeadIntel/0.1 (Business Intelligence Researcher; +https://github.com/example/leadintel; respectful crawler)"
+    }
+    if headers:
+        client_headers.update(headers)
+
+    transport = SafeAsyncHTTPTransport(verify=True)
+
+    return httpx.AsyncClient(
+        transport=transport,
+        timeout=timeout,
+        follow_redirects=True,
+        max_redirects=max_redirects,
+        trust_env=False,
+        event_hooks={"response": [validate_redirect_hook]},
+        headers=client_headers,
+    )

leadintel/services/website_analyzer.py

@@ -0,0 +1,213 @@
+"""
+Website Analyzer Service.
+
+Comprehensive website diagnostic service:
+- Checks availability, response time, and HTTP status.
+- Detects HTTP vs HTTPS and verifies SSL/TLS validity.
+- Extracts page title, meta description, and viewport configuration.
+- Detects H1, H2, and image alt presence.
+- Detects contact details (emails, phone numbers, contact forms).
+- Detects social media links.
+- Samples and checks internal links for 404/broken status where practical.
+- Protects against SSRF, respects robots.txt, and enforces size/rate limits.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import re
+from typing import Any, Optional
+from urllib.parse import urljoin, urlparse
+
+import httpx
+from bs4 import BeautifulSoup
+
+from .robots_checker import RobotsChecker
+from .social_finder import extract_social_profiles
+from .url_validator import create_safe_client, validate_and_normalize_url
+
+logger = logging.getLogger(__name__)
+
+USER_AGENT = "LeadIntel/0.1 (Business Intelligence Researcher; +https://github.com/example/leadintel; respectful crawler)"
+MAX_HTML_BYTES = 5_000_000  # 5MB max payload limit
+MAX_BROKEN_LINKS_CHECK = 10  # Maximum number of internal links to spot-check for broken links
+
+EMAIL_REGEX = re.compile(r"[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+")
+PHONE_REGEX = re.compile(r"(\+?\d{1,3}[-.\s]?)?\(?\d{2,4}\)?[-.\s]?\d{3,4}[-.\s]?\d{3,9}")
+
+
+async def analyze_target_website(
+    url: str,
+    check_broken_links: bool = True,
+    client: Optional[httpx.AsyncClient] = None,
+) -> dict[str, Any]:
+    """
+    Perform a complete website analysis on the given URL.
+
+    Returns structured diagnostics or error details.
+    """
+    norm_url = validate_and_normalize_url(url)
+
+    # 1. Check robots.txt
+    robots = RobotsChecker()
+    can_fetch, reason = await robots.can_fetch(norm_url)
+    if not can_fetch:
+        return {
+            "status": "blocked_by_robots",
+            "url": norm_url,
+            "available": False,
+            "error": reason or "Blocked by website robots.txt policy.",
+        }
+
+    should_close = False
+    if client is None:
+        client = create_safe_client(timeout=15.0)
+        should_close = True
+
+    try:
+        # Fetch page with streaming or byte cap
+        start_time = asyncio.get_event_loop().time()
+        try:
+            resp = await client.get(
+                norm_url,
+                headers={"User-Agent": USER_AGENT, "Accept": "text/html,application/xhtml+xml"},
+            )
+            elapsed = round(asyncio.get_event_loop().time() - start_time, 3)
+        except Exception as exc:
+            logger.debug("Failed to fetch target website %s: %s", norm_url, exc)
+            return {
+                "status": "error",
+                "url": norm_url,
+                "available": False,
+                "error": f"Connection failed or timed out: {exc}",
+            }
+
+        # Content length safeguard
+        raw_bytes = resp.content
+        if len(raw_bytes) > MAX_HTML_BYTES:
+            raw_bytes = raw_bytes[:MAX_HTML_BYTES]
+
+        html = raw_bytes.decode(resp.encoding or "utf-8", errors="replace")
+
+        # 2. SSL / Protocol Checks
+        final_url = str(resp.url)
+        is_https = final_url.startswith("https://")
+        ssl_valid = is_https  # If httpx connects over https with standard ssl verify=True without exception, it is valid
+
+        # 3. HTML Extraction via BeautifulSoup
+        soup = BeautifulSoup(html, "html.parser")
+
+        title_tag = soup.find("title")
+        title = title_tag.get_text().strip() if title_tag else None
+
+        desc_tag = soup.find("meta", attrs={"name": lambda x: x and x.lower() == "description"})
+        meta_description = desc_tag.get("content", "").strip() if desc_tag else None
+
+        viewport_tag = soup.find("meta", attrs={"name": lambda x: x and x.lower() == "viewport"})
+        has_mobile_viewport = bool(viewport_tag and "width=" in viewport_tag.get("content", "").lower())
+
+        # Headings & Media
+        h1_tags = [h.get_text().strip() for h in soup.find_all("h1") if h.get_text().strip()]
+        h2_tags = [h.get_text().strip() for h in soup.find_all("h2") if h.get_text().strip()]
+        images = soup.find_all("img")
+        images_missing_alt = [img for img in images if not img.get("alt") or not img.get("alt").strip()]
+
+        # 4. Contact Detection (Emails, Phone numbers, Forms)
+        contact_forms = soup.find_all("form")
+        has_contact_form = False
+        for f in contact_forms:
+            form_text = str(f).lower()
+            if any(term in form_text for term in ("contact", "email", "message", "inquiry", "send", "submit")):
+                has_contact_form = True
+                break
+
+        # Search for mailto / tel links first
+        emails: set[str] = set()
+        phones: set[str] = set()
+
+        for a in soup.find_all("a", href=True):
+            href = a["href"].strip()
+            if href.lower().startswith("mailto:"):
+                clean_email = href[7:].split("?")[0].strip()
+                if EMAIL_REGEX.match(clean_email):
+                    emails.add(clean_email)
+            elif href.lower().startswith("tel:"):
+                clean_tel = href[4:].split("?")[0].strip()
+                if len(clean_tel) >= 7:
+                    phones.add(clean_tel)
+
+        # Secondary regex search in body text (exclude scripts/styles)
+        for s in soup(["script", "style", "noscript"]):
+            s.decompose()
+        body_text = soup.get_text()
+
+        for em in EMAIL_REGEX.findall(body_text):
+            if not em.endswith((".png", ".jpg", ".jpeg", ".webp", ".svg", ".gif")):
+                emails.add(em)
+                if len(emails) >= 5:
+                    break
+
+        # 5. Social Profiles
+        social_profiles = extract_social_profiles(html, base_url=final_url)
+
+        # 6. Internal / Broken Links Spot-Check
+        base_domain = urlparse(final_url).netloc.lower()
+        internal_links_to_check: list[str] = []
+        for a in soup.find_all("a", href=True):
+            href = a["href"].strip()
+            if not href or href.startswith(("#", "javascript:", "mailto:", "tel:")):
+                continue
+            full_link = urljoin(final_url, href)
+            p_link = urlparse(full_link)
+            if p_link.netloc.lower() == base_domain and full_link != final_url:
+                if full_link not in internal_links_to_check:
+                    internal_links_to_check.append(full_link)
+            if len(internal_links_to_check) >= MAX_BROKEN_LINKS_CHECK:
+                break
+
+        broken_links: list[dict[str, Any]] = []
+        if check_broken_links and internal_links_to_check:
+            for link in internal_links_to_check[:MAX_BROKEN_LINKS_CHECK]:
+                try:
+                    head_resp = await client.head(link, timeout=5.0)
+                    if head_resp.status_code >= 400:
+                        broken_links.append({"url": link, "status_code": head_resp.status_code})
+                except Exception as e:
+                    broken_links.append({"url": link, "error": str(e)})
+
+        return {
+            "status": "success",
+            "url": final_url,
+            "available": resp.status_code < 400,
+            "http_status": resp.status_code,
+            "response_time_seconds": elapsed,
+            "protocol": {
+                "is_https": is_https,
+                "ssl_valid": ssl_valid,
+                "redirected": str(resp.url) != norm_url,
+            },
+            "metadata": {
+                "title": title,
+                "meta_description": meta_description,
+                "has_mobile_viewport": has_mobile_viewport,
+            },
+            "content_structure": {
+                "h1_headings": h1_tags,
+                "h2_headings_sample": h2_tags[:5],
+                "h2_count": len(h2_tags),
+                "images_total": len(images),
+                "images_missing_alt": len(images_missing_alt),
+            },
+            "contact_detection": {
+                "has_contact_form": has_contact_form,
+                "emails_found": list(emails)[:5],
+                "phones_found": list(phones)[:5],
+            },
+            "social_profiles": social_profiles,
+            "broken_internal_links": broken_links,
+        }
+
+    finally:
+        if should_close:
+            await client.aclose()