changedetection.io-osint-processor 0.0.1__py3-none-any.whl

changedetectionio_osint/steps/base.py

@@ -0,0 +1,76 @@
+"""
+Base class and interface for OSINT scan steps
+
+All scan steps should inherit from ScanStep and implement:
+- scan() method: Performs the actual scanning
+- format_results() method: Formats results for output
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Optional
+
+
+class ScanStep(ABC):
+    """
+    Base class for OSINT reconnaissance scan steps.
+
+    Each step represents an independent scan operation that can be
+    run serially or in parallel with other steps.
+    """
+
+    # Step name (used for section header in output)
+    # Override in subclass
+    name: str = "Unknown Step"
+
+    # Step order/priority (lower numbers run first in serial mode)
+    # Override in subclass
+    order: int = 100
+
+    @abstractmethod
+    async def scan(self, context: dict) -> Any:
+        """
+        Perform the scan operation.
+
+        Args:
+            context: Dictionary containing scan context:
+                - hostname: Target hostname
+                - ip_address: Resolved IP address
+                - url: Full target URL
+                - parsed_url: Parsed URL object
+                - dns_resolver: Configured DNS resolver
+                - proxy_url: Optional proxy URL
+                - watch_uuid: Watch UUID for status updates
+                - update_signal: Blinker signal for status updates
+
+        Returns:
+            Scan results (format depends on scan type)
+        """
+        pass
+
+    @abstractmethod
+    def format_results(self, results: Any) -> str:
+        """
+        Format scan results for output.
+
+        Args:
+            results: Results from scan() method
+
+        Returns:
+            Formatted string with section header and content
+        """
+        pass
+
+    def should_run(self, context: dict) -> bool:
+        """
+        Determine if this step should run based on context.
+
+        Override this method if the step should only run under certain conditions
+        (e.g., TLS scan only for HTTPS URLs).
+
+        Args:
+            context: Scan context dictionary
+
+        Returns:
+            True if step should run, False otherwise
+        """
+        return True

changedetectionio_osint/steps/bgp.py

@@ -0,0 +1,88 @@
+"""
+BGP Information Step
+Retrieves BGP/ASN information about the target IP
+"""
+
+import asyncio
+# SOCKS5 proxy support: BGP uses HTTP APIs, but proxy not currently implemented
+supports_socks5 = False
+from loguru import logger
+
+
+async def scan_bgp(ip_address, watch_uuid=None, update_signal=None):
+    """
+    Retrieve BGP/ASN information for target IP
+
+    Args:
+        ip_address: Target IP address
+        watch_uuid: Optional watch UUID for status updates
+        update_signal: Optional blinker signal for status updates
+
+    Returns:
+        dict: BGP/ASN information
+    """
+    if update_signal and watch_uuid:
+        update_signal.send(watch_uuid=watch_uuid, status="BGP")
+
+    def fetch_bgp_info():
+        import requests
+
+        bgp_data = {}
+
+        try:
+            # Use ip-api.com for basic ASN/ISP info (free, no key needed)
+            resp = requests.get(
+                f"http://ip-api.com/json/{ip_address}?fields=as,asname,isp,org,hosting",
+                timeout=3
+            )
+
+            if resp.status_code == 200:
+                data = resp.json()
+                if data.get('as'):
+                    bgp_data['asn'] = data['as']
+                if data.get('asname'):
+                    bgp_data['as_name'] = data['asname']
+                if data.get('isp'):
+                    bgp_data['isp'] = data['isp']
+                if data.get('org'):
+                    bgp_data['organization'] = data['org']
+                if data.get('hosting'):
+                    bgp_data['hosting'] = 'Yes' if data['hosting'] else 'No'
+
+        except Exception as e:
+            logger.error(f"BGP info lookup failed: {e}")
+
+        # Try to get more detailed BGP info from other sources
+        # Note: Most BGP APIs require authentication or have rate limits
+        # We'll add basic info here and can expand later
+
+        return bgp_data
+
+    try:
+        return await asyncio.to_thread(fetch_bgp_info)
+    except Exception as e:
+        logger.error(f"BGP scan failed: {e}")
+        return {}
+
+
+def format_bgp_results(bgp_data):
+    """Format BGP/ASN results for output"""
+    lines = []
+    lines.append("=== BGP / ASN Information ===")
+
+    if bgp_data:
+        if bgp_data.get('asn'):
+            lines.append(f"ASN: {bgp_data['asn']}")
+        if bgp_data.get('as_name'):
+            lines.append(f"AS Name: {bgp_data['as_name']}")
+        if bgp_data.get('isp'):
+            lines.append(f"ISP: {bgp_data['isp']}")
+        if bgp_data.get('organization'):
+            lines.append(f"Organization: {bgp_data['organization']}")
+        if bgp_data.get('hosting'):
+            lines.append(f"Hosting/Datacenter: {bgp_data['hosting']}")
+    else:
+        lines.append("BGP information not available")
+
+    lines.append("")
+    return '\n'.join(lines)

changedetectionio_osint/steps/dns.py

@@ -0,0 +1,147 @@
+"""
+DNS Reconnaissance Step
+Performs comprehensive DNS queries using configured DNS server
+"""
+
+import asyncio
+from loguru import logger
+
+# SOCKS5 proxy support
+# DNS can use TCP on port 53, which works through SOCKS5
+# We force TCP mode when proxy is configured
+supports_socks5 = True
+
+
+async def scan_dns(hostname, dns_resolver, proxy_url=None, watch_uuid=None, update_signal=None):
+    """
+    Perform DNS reconnaissance on hostname
+
+    Args:
+        hostname: Target hostname to query
+        dns_resolver: Configured dns.resolver.Resolver instance
+        proxy_url: Optional SOCKS5 proxy URL (forces DNS-over-TCP)
+        watch_uuid: Optional watch UUID for status updates
+        update_signal: Optional blinker signal for status updates
+
+    Returns:
+        dict: DNS results with record types as keys
+    """
+    if update_signal and watch_uuid:
+        update_signal.send(watch_uuid=watch_uuid, status="DNS")
+
+    def query_dns():
+        import dns.query
+        import dns.message
+        import dns.rdatatype
+
+        results = {}
+        record_types = ['A', 'AAAA', 'MX', 'NS', 'TXT', 'SOA', 'CAA']
+
+        # Get DNS server from resolver
+        dns_server = dns_resolver.nameservers[0] if dns_resolver.nameservers else '8.8.8.8'
+
+        for rtype in record_types:
+            try:
+                # Create DNS query message
+                query = dns.message.make_query(hostname, rtype)
+
+                # If proxy is configured, use TCP (works through SOCKS5)
+                # Otherwise, use UDP (faster)
+                if proxy_url and proxy_url.strip():
+                    try:
+                        # DNS-over-TCP through SOCKS5 proxy
+                        from python_socks.sync import Proxy
+                        from urllib.parse import urlparse
+                        import socket as sock_module
+
+                        # Parse proxy URL
+                        proxy = Proxy.from_url(proxy_url)
+
+                        # Create SOCKS5 connection to DNS server
+                        socks_socket = proxy.connect(dest_host=dns_server, dest_port=53)
+
+                        # Send DNS query over TCP through SOCKS5
+                        response = dns.query.tcp(query, dns_server, sock=socks_socket, timeout=5)
+
+                        # Close socket
+                        socks_socket.close()
+
+                    except ImportError:
+                        # CRITICAL: Do NOT fallback to direct connection - would leak real IP!
+                        logger.error("SOCKS5 proxy configured but 'python-socks' not installed - DNS query BLOCKED to prevent IP leak")
+                        raise Exception("DNS-over-SOCKS5 requires 'python-socks' package. Install with: pip install 'python-socks[asyncio]'")
+                    except Exception as e:
+                        # CRITICAL: Do NOT fallback to direct connection - would leak real IP!
+                        logger.error(f"DNS-over-TCP via SOCKS5 failed for {rtype}: {e} - query BLOCKED to prevent IP leak")
+                        raise
+                else:
+                    # Direct UDP query (no proxy)
+                    response = dns.query.udp(query, dns_server, timeout=5)
+
+                # Parse response
+                results[rtype] = []
+                for rrset in response.answer:
+                    for rdata in rrset:
+                        if rtype == 'MX':
+                            results[rtype].append(f"{rdata.preference} {rdata.exchange}")
+                        elif rtype == 'SOA':
+                            results[rtype].append(f"{rdata.mname} {rdata.rname}")
+                        elif rtype == 'CAA':
+                            results[rtype].append(f"{rdata.flags} {rdata.tag} {rdata.value}")
+                        else:
+                            results[rtype].append(str(rdata))
+
+            except Exception as e:
+                logger.debug(f"DNS query for {rtype} failed: {e}")
+                pass
+
+        return results
+
+    return await asyncio.to_thread(query_dns)
+
+
+def format_dns_results(dns_results):
+    """Format DNS results for output"""
+    lines = []
+    lines.append("=== DNS Records ===")
+
+    if dns_results:
+        for rtype, records in sorted(dns_results.items()):
+            if records:
+                lines.append(f"{rtype} Records:")
+
+                # Sort records based on type
+                if rtype == 'MX':
+                    # MX records: sort by priority (numeric), then alphabetically
+                    # Format: "10 mail.example.com."
+                    def mx_sort_key(mx_record):
+                        try:
+                            parts = mx_record.split(' ', 1)
+                            priority = int(parts[0])
+                            server = parts[1] if len(parts) > 1 else ''
+                            return (priority, server)
+                        except:
+                            return (999999, mx_record)
+
+                    sorted_records = sorted(records, key=mx_sort_key)
+
+                elif rtype == 'TXT':
+                    # TXT records: sort alphabetically
+                    sorted_records = sorted(records)
+
+                elif rtype == 'NS':
+                    # NS records: sort alphabetically for consistent ordering
+                    sorted_records = sorted(records)
+
+                else:
+                    # Other records: keep original order
+                    sorted_records = records
+
+                # Output sorted records
+                for record in sorted_records:
+                    lines.append(f"  {record}")
+    else:
+        lines.append("No DNS records found")
+
+    lines.append("")
+    return '\n'.join(lines)

changedetectionio_osint/steps/dns_scan.py

@@ -0,0 +1,88 @@
+"""
+DNS Reconnaissance Step
+"""
+
+import asyncio
+from loguru import logger
+from .base import ScanStep
+from .registry import register_step
+
+
+@register_step
+class DNSScanStep(ScanStep):
+    """DNS record queries (A, AAAA, MX, NS, TXT, SOA, CAA)"""
+
+    name = "DNS Records"
+    order = 10
+
+    async def scan(self, context: dict):
+        """Perform DNS reconnaissance"""
+        hostname = context['hostname']
+        dns_resolver = context['dns_resolver']
+        watch_uuid = context.get('watch_uuid')
+        update_signal = context.get('update_signal')
+
+        if update_signal and watch_uuid:
+            update_signal.send(watch_uuid=watch_uuid, status="DNS")
+
+        def query_dns():
+            results = {}
+            record_types = ['A', 'AAAA', 'MX', 'NS', 'TXT', 'SOA', 'CAA']
+
+            for rtype in record_types:
+                try:
+                    answers = dns_resolver.resolve(hostname, rtype)
+                    results[rtype] = []
+                    for rdata in answers:
+                        if rtype == 'MX':
+                            results[rtype].append(f"{rdata.preference} {rdata.exchange}")
+                        elif rtype == 'SOA':
+                            results[rtype].append(f"{rdata.mname} {rdata.rname}")
+                        elif rtype == 'CAA':
+                            results[rtype].append(f"{rdata.flags} {rdata.tag} {rdata.value}")
+                        else:
+                            results[rtype].append(str(rdata))
+                except Exception as e:
+                    logger.debug(f"DNS query for {rtype} failed: {e}")
+                    pass
+
+            return results
+
+        return await asyncio.to_thread(query_dns)
+
+    def format_results(self, dns_results):
+        """Format DNS results"""
+        lines = []
+        lines.append("=== DNS Records ===")
+
+        if dns_results and not isinstance(dns_results, Exception):
+            for rtype, records in sorted(dns_results.items()):
+                if records:
+                    lines.append(f"{rtype} Records:")
+
+                    # Sort records based on type for consistent output
+                    if rtype == 'MX':
+                        # MX records: sort by priority (numeric), then alphabetically
+                        def mx_sort_key(mx_record):
+                            try:
+                                parts = mx_record.split(' ', 1)
+                                priority = int(parts[0])
+                                server = parts[1] if len(parts) > 1 else ''
+                                return (priority, server)
+                            except:
+                                return (999999, mx_record)
+                        sorted_records = sorted(records, key=mx_sort_key)
+                    elif rtype in ['NS', 'TXT']:
+                        # NS and TXT records: sort alphabetically for consistent ordering
+                        sorted_records = sorted(records)
+                    else:
+                        # Other records: keep original order
+                        sorted_records = records
+
+                    for record in sorted_records:
+                        lines.append(f"  {record}")
+        else:
+            lines.append("No DNS records found")
+
+        lines.append("")
+        return '\n'.join(lines)

changedetectionio_osint/steps/dnssec.py

@@ -0,0 +1,260 @@
+"""
+DNSSEC Validation Step
+Validates DNSSEC signatures and chain of trust for domain security
+"""
+
+import asyncio
+# SOCKS5 proxy support: Requires DNS-over-TCP implementation (TODO: use dns.query.tcp with SOCKS5 socket)
+supports_socks5 = False
+from loguru import logger
+from datetime import datetime
+
+
+async def scan_dnssec(hostname, dns_resolver, watch_uuid=None, update_signal=None):
+    """
+    Perform DNSSEC validation
+
+    Args:
+        hostname: Target hostname to validate
+        dns_resolver: Configured dns.resolver.Resolver instance
+        watch_uuid: Optional watch UUID for status updates
+        update_signal: Optional blinker signal for status updates
+
+    Returns:
+        dict: DNSSEC validation results
+    """
+    if update_signal and watch_uuid:
+        update_signal.send(watch_uuid=watch_uuid, status="DNSSEC")
+
+    def query_dnssec():
+        import dns.dnssec
+        import dns.name
+
+        results = {
+            'dnssec_enabled': False,
+            'has_dnskey': False,
+            'has_ds': False,
+            'has_rrsig': False,
+            'validation_status': 'unknown',
+            'algorithm': None,
+            'key_count': 0,
+            'rrsig_count': 0,
+            'signatures': [],
+            'keys': [],
+            'error': None
+        }
+
+        try:
+            domain_name = dns.name.from_text(hostname)
+
+            # === Check for DNSKEY records (public keys) ===
+            try:
+                dnskey_answers = dns_resolver.resolve(hostname, 'DNSKEY')
+                results['has_dnskey'] = True
+                results['key_count'] = len(dnskey_answers)
+
+                for rdata in dnskey_answers:
+                    # DNSKEY flags: 256 = Zone Signing Key (ZSK), 257 = Key Signing Key (KSK)
+                    key_type = "KSK (Key Signing Key)" if rdata.flags == 257 else "ZSK (Zone Signing Key)" if rdata.flags == 256 else f"Unknown (flags={rdata.flags})"
+
+                    # Algorithm mapping (common ones)
+                    alg_names = {
+                        5: "RSA/SHA-1",
+                        7: "RSASHA1-NSEC3-SHA1",
+                        8: "RSA/SHA-256",
+                        10: "RSA/SHA-512",
+                        13: "ECDSA Curve P-256 with SHA-256",
+                        14: "ECDSA Curve P-384 with SHA-384",
+                        15: "Ed25519",
+                        16: "Ed448"
+                    }
+                    algorithm = alg_names.get(rdata.algorithm, f"Algorithm {rdata.algorithm}")
+
+                    results['keys'].append({
+                        'type': key_type,
+                        'algorithm': algorithm,
+                        'flags': rdata.flags,
+                        'protocol': rdata.protocol,
+                        'key_tag': dns.dnssec.key_id(rdata)
+                    })
+
+                    if not results['algorithm']:
+                        results['algorithm'] = algorithm
+
+                logger.debug(f"Found {results['key_count']} DNSKEY records for {hostname}")
+
+            except Exception as e:
+                logger.debug(f"No DNSKEY records found: {e}")
+
+            # === Check for DS records (delegation signer) ===
+            # DS records exist at the parent zone, proving the child is signed
+            try:
+                ds_answers = dns_resolver.resolve(hostname, 'DS')
+                results['has_ds'] = len(ds_answers) > 0
+                logger.debug(f"Found DS records for {hostname}")
+            except Exception as e:
+                logger.debug(f"No DS records found: {e}")
+
+            # === Check for RRSIG records (signatures) ===
+            # RRSIG records sign other record types (A, AAAA, etc.)
+            try:
+                # Try to get RRSIG for A records
+                a_answers = dns_resolver.resolve(hostname, 'A')
+                # Get the RRSIG that covers the A records
+                rrsig_answers = dns_resolver.resolve(hostname, 'RRSIG', rdtype=dns.rdatatype.A)
+                results['has_rrsig'] = True
+                results['rrsig_count'] = len(rrsig_answers)
+
+                for rdata in rrsig_answers:
+                    # Parse signature timing
+                    inception_time = datetime.fromtimestamp(rdata.inception)
+                    expiration_time = datetime.fromtimestamp(rdata.expiration)
+                    now = datetime.now()
+
+                    # Check if signature is currently valid
+                    is_valid = inception_time <= now <= expiration_time
+
+                    alg_names = {
+                        5: "RSA/SHA-1", 7: "RSASHA1-NSEC3-SHA1", 8: "RSA/SHA-256",
+                        10: "RSA/SHA-512", 13: "ECDSA P-256", 14: "ECDSA P-384",
+                        15: "Ed25519", 16: "Ed448"
+                    }
+                    algorithm = alg_names.get(rdata.algorithm, f"Algorithm {rdata.algorithm}")
+
+                    results['signatures'].append({
+                        'type_covered': dns.rdatatype.to_text(rdata.type_covered),
+                        'algorithm': algorithm,
+                        'signer': str(rdata.signer),
+                        'key_tag': rdata.key_tag,
+                        'inception': inception_time.strftime('%Y-%m-%d %H:%M:%S'),
+                        'expiration': expiration_time.strftime('%Y-%m-%d %H:%M:%S'),
+                        'valid': is_valid,
+                        'days_until_expiry': (expiration_time - now).days if is_valid else None
+                    })
+
+                logger.debug(f"Found {results['rrsig_count']} RRSIG records for {hostname}")
+
+            except Exception as e:
+                logger.debug(f"No RRSIG records found: {e}")
+
+            # === Determine overall DNSSEC status ===
+            if results['has_dnskey'] and results['has_rrsig']:
+                results['dnssec_enabled'] = True
+                # Full chain validation requires DS records in parent zone
+                if results['has_ds']:
+                    results['validation_status'] = 'secure (full chain)'
+                else:
+                    results['validation_status'] = 'secure (no DS records in parent zone)'
+            elif results['has_dnskey'] or results['has_rrsig']:
+                results['dnssec_enabled'] = True
+                results['validation_status'] = 'partial (incomplete DNSSEC)'
+            else:
+                results['dnssec_enabled'] = False
+                results['validation_status'] = 'unsigned (no DNSSEC)'
+
+        except Exception as e:
+            logger.error(f"DNSSEC validation error: {e}")
+            results['error'] = str(e)
+            results['validation_status'] = 'error'
+
+        return results
+
+    return await asyncio.to_thread(query_dnssec)
+
+
+def format_dnssec_results(dnssec_results):
+    """Format DNSSEC validation results for output"""
+    lines = []
+    lines.append("=== DNSSEC Validation ===")
+
+    if not dnssec_results:
+        lines.append("DNSSEC validation failed")
+        lines.append("")
+        return '\n'.join(lines)
+
+    if dnssec_results.get('error'):
+        lines.append(f"Error: {dnssec_results['error']}")
+        lines.append("")
+        return '\n'.join(lines)
+
+    # Overall Status
+    lines.append("")
+    status = dnssec_results.get('validation_status', 'unknown')
+    if 'secure' in status:
+        lines.append(f"Status: ✓ DNSSEC Enabled - {status}")
+        lines.append("Security: ✓ DNS responses are cryptographically signed")
+    elif 'partial' in status:
+        lines.append(f"Status: ⚠ {status}")
+        lines.append("Security: ⚠ DNSSEC is partially configured")
+    elif 'unsigned' in status:
+        lines.append(f"Status: ✗ {status}")
+        lines.append("Security: ✗ DNS responses are NOT signed (vulnerable to DNS spoofing)")
+    else:
+        lines.append(f"Status: ? {status}")
+
+    # DNSKEY Records (Public Keys)
+    if dnssec_results.get('has_dnskey'):
+        lines.append("")
+        lines.append(f"DNSKEY Records: ✓ Found {dnssec_results.get('key_count', 0)} key(s)")
+        for idx, key in enumerate(dnssec_results.get('keys', []), 1):
+            lines.append(f"  Key {idx}:")
+            lines.append(f"    Type: {key['type']}")
+            lines.append(f"    Algorithm: {key['algorithm']}")
+            lines.append(f"    Key Tag: {key['key_tag']}")
+    else:
+        lines.append("")
+        lines.append("DNSKEY Records: ✗ Not found")
+
+    # DS Records (Delegation Signer)
+    if dnssec_results.get('has_ds'):
+        lines.append("")
+        lines.append("DS Records: ✓ Found in parent zone")
+        lines.append("  Chain of Trust: ✓ Domain is properly delegated from parent")
+    else:
+        lines.append("")
+        lines.append("DS Records: ✗ Not found in parent zone")
+        if dnssec_results.get('has_dnskey'):
+            lines.append("  Chain of Trust: ⚠ Keys exist but not published to parent zone")
+
+    # RRSIG Records (Signatures)
+    if dnssec_results.get('has_rrsig'):
+        lines.append("")
+        lines.append(f"RRSIG Records: ✓ Found {dnssec_results.get('rrsig_count', 0)} signature(s)")
+
+        for idx, sig in enumerate(dnssec_results.get('signatures', []), 1):
+            lines.append(f"  Signature {idx}:")
+            lines.append(f"    Covers: {sig['type_covered']} records")
+            lines.append(f"    Algorithm: {sig['algorithm']}")
+            lines.append(f"    Signer: {sig['signer']}")
+            lines.append(f"    Key Tag: {sig['key_tag']}")
+            lines.append(f"    Valid From: {sig['inception']}")
+            lines.append(f"    Expires: {sig['expiration']}")
+
+            if sig['valid']:
+                days = sig.get('days_until_expiry')
+                if days is not None:
+                    if days <= 7:
+                        lines.append(f"    Status: ⚠ Valid but expires in {days} days")
+                    else:
+                        lines.append(f"    Status: ✓ Valid ({days} days remaining)")
+            else:
+                lines.append("    Status: ✗ EXPIRED or not yet valid")
+    else:
+        lines.append("")
+        lines.append("RRSIG Records: ✗ Not found")
+
+    # Summary and Recommendations
+    lines.append("")
+    lines.append("DNSSEC Summary:")
+    if dnssec_results.get('dnssec_enabled'):
+        if dnssec_results.get('has_ds'):
+            lines.append("  ✓ Full DNSSEC deployment with proper chain of trust")
+        else:
+            lines.append("  ⚠ DNSSEC configured but DS records not published to parent")
+            lines.append("  Recommendation: Publish DS records to parent zone for full validation")
+    else:
+        lines.append("  ✗ DNSSEC not configured")
+        lines.append("  Recommendation: Enable DNSSEC to protect against DNS spoofing/cache poisoning")
+
+    lines.append("")
+    return '\n'.join(lines)