changedetection.io-osint-processor 0.0.1__py3-none-any.whl

changedetectionio_osint/steps/mac_lookup.py

@@ -0,0 +1,209 @@
+"""
+MAC Address Lookup Step
+Gets MAC address from ARP cache for local network targets
+"""
+
+import asyncio
+# SOCKS5 proxy support: MAC address lookups are local network only (Layer 2)
+supports_socks5 = False
+import re
+import subprocess
+from loguru import logger
+
+
+async def scan_mac(ip_address, watch_uuid=None, update_signal=None):
+    """
+    Lookup MAC address from ARP cache
+
+    Args:
+        ip_address: Target IP address
+        watch_uuid: Optional watch UUID for status updates
+        update_signal: Optional blinker signal for status updates
+
+    Returns:
+        dict: MAC address and vendor info
+    """
+    if update_signal and watch_uuid:
+        update_signal.send(watch_uuid=watch_uuid, status="MAC")
+
+    def get_mac_from_arp():
+        """Try to get MAC address from ARP cache"""
+        result = {
+            'mac_address': None,
+            'vendor': None,
+            'method': None
+        }
+
+        try:
+            # Try reading /proc/net/arp on Linux (no root needed)
+            try:
+                with open('/proc/net/arp', 'r') as f:
+                    arp_data = f.read()
+
+                for line in arp_data.split('\n')[1:]:  # Skip header
+                    if ip_address in line:
+                        parts = line.split()
+                        if len(parts) >= 4:
+                            mac = parts[3]
+                            if mac != '00:00:00:00:00:00' and mac != '<incomplete>':
+                                result['mac_address'] = mac.upper()
+                                result['method'] = '/proc/net/arp'
+                                logger.debug(f"Found MAC {mac} for {ip_address} in /proc/net/arp")
+                                break
+            except (FileNotFoundError, PermissionError):
+                logger.debug("/proc/net/arp not available, trying arp command")
+
+            # Fallback: Try 'ip neigh' command (Linux)
+            if not result['mac_address']:
+                try:
+                    output = subprocess.check_output(
+                        ['ip', 'neigh', 'show', ip_address],
+                        stderr=subprocess.DEVNULL,
+                        timeout=2
+                    ).decode('utf-8')
+
+                    # Parse: 192.168.1.1 dev eth0 lladdr aa:bb:cc:dd:ee:ff REACHABLE
+                    mac_match = re.search(r'lladdr\s+([0-9a-fA-F:]{17})', output)
+                    if mac_match:
+                        result['mac_address'] = mac_match.group(1).upper()
+                        result['method'] = 'ip neigh'
+                        logger.debug(f"Found MAC via 'ip neigh': {result['mac_address']}")
+                except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired):
+                    logger.debug("'ip neigh' command failed")
+
+            # Fallback: Try 'arp' command (Linux/Mac)
+            if not result['mac_address']:
+                try:
+                    output = subprocess.check_output(
+                        ['arp', '-n', ip_address],
+                        stderr=subprocess.DEVNULL,
+                        timeout=2
+                    ).decode('utf-8')
+
+                    # Parse: 192.168.1.1 ether aa:bb:cc:dd:ee:ff C eth0
+                    mac_match = re.search(r'([0-9a-fA-F:]{17})', output)
+                    if mac_match:
+                        result['mac_address'] = mac_match.group(1).upper()
+                        result['method'] = 'arp command'
+                        logger.debug(f"Found MAC via 'arp': {result['mac_address']}")
+                except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired):
+                    logger.debug("'arp' command failed")
+
+            # Lookup vendor if we found a MAC
+            if result['mac_address']:
+                result['vendor'] = lookup_mac_vendor(result['mac_address'])
+
+        except Exception as e:
+            logger.debug(f"MAC lookup failed: {e}")
+
+        return result
+
+    return await asyncio.to_thread(get_mac_from_arp)
+
+
+def lookup_mac_vendor(mac_address):
+    """
+    Lookup MAC vendor from OUI (Organizationally Unique Identifier)
+
+    Args:
+        mac_address: MAC address in format AA:BB:CC:DD:EE:FF
+
+    Returns:
+        str: Vendor name or None
+    """
+    try:
+        # Try using mac-vendor-lookup library if available
+        try:
+            from mac_vendor_lookup import MacLookup
+            mac = MacLookup()
+            vendor = mac.lookup(mac_address)
+            return vendor
+        except ImportError:
+            logger.debug("mac-vendor-lookup not installed, using fallback")
+        except Exception as e:
+            logger.debug(f"mac-vendor-lookup failed: {e}")
+
+        # Fallback: Basic OUI lookup using common vendors
+        # Just first 3 bytes (OUI)
+        oui = mac_address[:8].replace(':', '').upper()
+
+        # Common vendor OUIs (just a small sample - full database has 40k+ entries)
+        common_ouis = {
+            '00005E': 'IANA/ICANN',
+            '00000C': 'Cisco Systems',
+            '000D3A': 'Cisco Systems',
+            '001A2B': 'Cisco Systems',
+            '00503E': 'Cisco Systems',
+            '001B0D': 'D-Link',
+            '0018E7': 'TP-Link',
+            '0C8268': 'TP-Link',
+            'F4EC38': 'TP-Link',
+            '001C7F': 'NETGEAR',
+            '002275': 'NETGEAR',
+            'A0CCEC': 'NETGEAR',
+            'C83A35': 'NETGEAR',
+            '000C29': 'VMware',
+            '005056': 'VMware',
+            '000569': 'VMware',
+            '001C14': 'VMware',
+            '080027': 'Oracle VirtualBox',
+            '525400': 'QEMU Virtual NIC',
+            '000000': 'Xerox Corporation',
+            '001D7E': 'Raspberry Pi Foundation',
+            'B827EB': 'Raspberry Pi Foundation',
+            'DC4A3E': 'Raspberry Pi Foundation',
+            'E45F01': 'Raspberry Pi Foundation',
+            '000A27': 'Apple',
+            '000D93': 'Apple',
+            '001124': 'Apple',
+            '0016CB': 'Apple',
+            '001E52': 'Apple',
+            '002332': 'Apple',
+            '002436': 'Apple',
+            '002500': 'Apple',
+            '0026BB': 'Apple',
+            '00DB70': 'Apple',
+            '00E091': 'Microsoft',
+            '000BDB': 'Microsoft',
+            '001DD8': 'Microsoft',
+            '00155D': 'Microsoft Hyper-V',
+            '74D435': 'Google',
+            'F4F5D8': 'Google',
+            '3C5A37': 'Google',
+            '000C76': 'Intel',
+            '001E67': 'Intel',
+            '0024D7': 'Intel',
+            '7085C2': 'Intel',
+            '001B21': 'Intel',
+            '002170': 'Dell',
+            '0026B9': 'Dell',
+            'D4AE52': 'Dell',
+            'F04DA2': 'Dell',
+            '002564': 'Dell',
+        }
+
+        return common_ouis.get(oui[:6], 'Unknown Vendor')
+
+    except Exception as e:
+        logger.debug(f"Vendor lookup failed: {e}")
+        return None
+
+
+def format_mac_results(mac_data):
+    """Format MAC address results for output"""
+    lines = []
+    lines.append("=== MAC Address (Local Network) ===")
+
+    if mac_data and mac_data.get('mac_address'):
+        lines.append(f"MAC Address: {mac_data['mac_address']}")
+        if mac_data.get('vendor'):
+            lines.append(f"Vendor: {mac_data['vendor']}")
+        lines.append(f"Detection Method: {mac_data.get('method', 'unknown')}")
+        lines.append("")
+        lines.append("Note: MAC address only available for local network devices")
+    else:
+        lines.append("Not available (target not in ARP cache)")
+        lines.append("Note: MAC addresses only visible for devices on the same local network")
+
+    lines.append("")
+    return '\n'.join(lines)

changedetectionio_osint/steps/os_detection.py

@@ -0,0 +1,245 @@
+"""
+OS Detection Step
+Performs operating system fingerprinting when raw socket permissions available
+"""
+
+import asyncio
+# SOCKS5 proxy support: OS detection requires raw sockets/ICMP
+supports_socks5 = False
+import socket
+import struct
+from loguru import logger
+
+
+def check_raw_socket_permission():
+    """
+    Check if we have raw socket permissions (requires root or CAP_NET_RAW)
+
+    Returns:
+        bool: True if raw sockets are available
+    """
+    try:
+        # Try to create a raw ICMP socket
+        s = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_ICMP)
+        s.close()
+        return True
+    except PermissionError:
+        return False
+    except OSError as e:
+        # Other errors (e.g., protocol not supported)
+        logger.debug(f"Raw socket check failed: {e}")
+        return False
+
+
+async def scan_os(ip_address, watch_uuid=None, update_signal=None):
+    """
+    Perform OS detection/fingerprinting
+
+    Args:
+        ip_address: Target IP address
+        watch_uuid: Optional watch UUID for status updates
+        update_signal: Optional blinker signal for status updates
+
+    Returns:
+        dict: OS detection results
+    """
+    if update_signal and watch_uuid:
+        update_signal.send(watch_uuid=watch_uuid, status="OS Detection")
+
+    def detect_os():
+        result = {
+            'has_raw_socket': False,
+            'os_guess': None,
+            'confidence': None,
+            'ttl': None,
+            'method': 'passive'
+        }
+
+        # Check for raw socket permissions
+        result['has_raw_socket'] = check_raw_socket_permission()
+
+        if result['has_raw_socket']:
+            logger.info("Raw socket access available - attempting active OS fingerprinting")
+            result['method'] = 'active'
+
+            # Active fingerprinting with raw sockets
+            try:
+                # Send ICMP ping and analyze TTL
+                ttl = get_ttl_via_ping(ip_address)
+                if ttl:
+                    result['ttl'] = ttl
+                    result['os_guess'], result['confidence'] = guess_os_from_ttl(ttl)
+            except Exception as e:
+                logger.debug(f"Active OS detection failed: {e}")
+                result['method'] = 'passive'
+        else:
+            logger.debug("No raw socket access - OS detection limited to passive methods")
+
+        return result
+
+    return await asyncio.to_thread(detect_os)
+
+
+def get_ttl_via_ping(ip_address, timeout=2):
+    """
+    Send ICMP ping and extract TTL from response
+
+    Args:
+        ip_address: Target IP
+        timeout: Socket timeout in seconds
+
+    Returns:
+        int: TTL value or None
+    """
+    try:
+        # Create raw ICMP socket
+        icmp = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_ICMP)
+        icmp.settimeout(timeout)
+
+        # ICMP Echo Request packet
+        # Type=8 (Echo Request), Code=0, Checksum=0 (will calculate), ID=0, Seq=0
+        packet_id = 12345
+        packet_seq = 1
+
+        # Build ICMP header (type, code, checksum, id, sequence)
+        header = struct.pack('!BBHHH', 8, 0, 0, packet_id, packet_seq)
+        data = b'OSINT' * 10  # 50 bytes of data
+
+        # Calculate checksum
+        checksum = calculate_checksum(header + data)
+        header = struct.pack('!BBHHH', 8, 0, checksum, packet_id, packet_seq)
+
+        packet = header + data
+
+        # Send packet
+        icmp.sendto(packet, (ip_address, 0))
+
+        # Receive response
+        try:
+            reply, addr = icmp.recvfrom(1024)
+
+            # Extract TTL from IP header (9th byte)
+            ttl = reply[8]
+
+            icmp.close()
+            logger.debug(f"Received ICMP reply from {ip_address} with TTL={ttl}")
+            return ttl
+
+        except socket.timeout:
+            logger.debug(f"ICMP ping timeout for {ip_address}")
+            icmp.close()
+            return None
+
+    except Exception as e:
+        logger.debug(f"ICMP ping failed: {e}")
+        return None
+
+
+def calculate_checksum(data):
+    """Calculate ICMP checksum"""
+    if len(data) % 2 == 1:
+        data += b'\x00'
+
+    s = sum(struct.unpack('!%dH' % (len(data) // 2), data))
+    s = (s >> 16) + (s & 0xffff)
+    s += s >> 16
+    s = ~s & 0xffff
+
+    return s
+
+
+def guess_os_from_ttl(ttl):
+    """
+    Guess OS based on TTL value
+
+    Common initial TTL values:
+    - Linux/Unix: 64
+    - Windows: 128
+    - Cisco/Network devices: 255
+    - Some BSD: 255
+
+    Args:
+        ttl: TTL value from packet
+
+    Returns:
+        tuple: (os_guess, confidence)
+    """
+    # Calculate likely initial TTL (common values: 32, 64, 128, 255)
+    possible_initial_ttls = [32, 64, 128, 255]
+
+    # Find the smallest initial TTL that's >= observed TTL
+    initial_ttl = None
+    for value in possible_initial_ttls:
+        if ttl <= value:
+            initial_ttl = value
+            break
+
+    if not initial_ttl:
+        return ("Unknown (TTL too high)", "low")
+
+    hop_count = initial_ttl - ttl
+
+    # Make educated guesses based on initial TTL
+    if initial_ttl == 64:
+        # Most likely Linux/Unix
+        if hop_count <= 5:
+            return ("Linux/Unix", "high")
+        else:
+            return ("Linux/Unix", "medium")
+
+    elif initial_ttl == 128:
+        # Most likely Windows
+        if hop_count <= 5:
+            return ("Windows", "high")
+        else:
+            return ("Windows", "medium")
+
+    elif initial_ttl == 255:
+        # Could be Cisco, BSD, or other network device
+        if hop_count <= 5:
+            return ("Cisco/Network Device/BSD", "medium")
+        else:
+            return ("Cisco/Network Device/BSD", "low")
+
+    elif initial_ttl == 32:
+        # Less common, older Windows or some embedded systems
+        return ("Windows 95/98 or Embedded Device", "low")
+
+    return ("Unknown", "low")
+
+
+def format_os_results(os_data):
+    """Format OS detection results for output"""
+    lines = []
+    lines.append("=== Operating System Detection ===")
+
+    if not os_data:
+        lines.append("OS detection unavailable")
+        lines.append("")
+        return '\n'.join(lines)
+
+    if os_data.get('has_raw_socket'):
+        lines.append("Method: Active fingerprinting (raw sockets available)")
+    else:
+        lines.append("Method: Passive analysis (no raw socket permissions)")
+        lines.append("Note: Run as root or grant CAP_NET_RAW for active OS fingerprinting")
+
+    lines.append("")
+
+    if os_data.get('ttl'):
+        lines.append(f"TTL: {os_data['ttl']}")
+
+    if os_data.get('os_guess'):
+        confidence = os_data.get('confidence', 'unknown')
+        lines.append(f"OS Guess: {os_data['os_guess']} (confidence: {confidence})")
+    else:
+        lines.append("OS: Unable to determine")
+
+    if not os_data.get('has_raw_socket'):
+        lines.append("")
+        lines.append("💡 Tip: For more accurate OS detection, run changedetection.io with:")
+        lines.append("   • Root privileges, or")
+        lines.append("   • CAP_NET_RAW capability: sudo setcap cap_net_raw+ep /path/to/python")
+
+    lines.append("")
+    return '\n'.join(lines)

changedetectionio_osint/steps/portscan.py

@@ -0,0 +1,113 @@
+"""
+Port Scanning Step
+Fast asyncio-based TCP connect port scanning
+"""
+
+import asyncio
+# SOCKS5 proxy support: Port scanning not compatible with SOCKS5
+supports_socks5 = False
+from loguru import logger
+
+# ============================================================================
+# CONFIGURATION
+# ============================================================================
+
+# Port list extracted from /etc/services (TCP ports 1-10000)
+# This hardcoded list works on all platforms (Windows, Linux, etc.)
+COMMON_PORTS = [
+    1, 7, 9, 11, 13, 15, 17, 19, 20, 21, 22, 23, 25, 37, 43, 49, 53, 70, 79, 80,
+    88, 102, 104, 106, 110, 111, 113, 119, 135, 139, 143, 161, 162, 163, 164, 174,
+    179, 199, 209, 210, 345, 346, 369, 370, 389, 427, 443, 444, 445, 464, 465, 487,
+    512, 513, 514, 515, 538, 540, 543, 544, 548, 554, 563, 587, 607, 628, 631, 636,
+    646, 655, 706, 749, 750, 751, 754, 775, 777, 783, 853, 871, 873, 989, 990, 992,
+    993, 995, 1080, 1093, 1094, 1099, 1127, 1178, 1194, 1236, 1313, 1314, 1352, 1433,
+    1524, 1645, 1646, 1649, 1677, 1812, 1813, 2000, 2049, 2086, 2101, 2119, 2121, 2135,
+    2401, 2430, 2431, 2432, 2433, 2583, 2600, 2601
+]
+
+# Service name mapping for common ports (extracted from /etc/services)
+PORT_SERVICES = {
+    1: 'tcpmux', 7: 'echo', 9: 'discard', 11: 'systat', 13: 'daytime', 15: 'netstat',
+    17: 'qotd', 19: 'chargen', 20: 'ftp-data', 21: 'ftp', 22: 'ssh', 23: 'telnet',
+    25: 'smtp', 37: 'time', 43: 'whois', 49: 'tacacs', 53: 'domain', 70: 'gopher',
+    79: 'finger', 80: 'http', 88: 'kerberos', 102: 'iso-tsap', 104: 'acr-nema',
+    106: 'poppassd', 110: 'pop3', 111: 'sunrpc', 113: 'auth', 119: 'nntp', 135: 'epmap',
+    139: 'netbios-ssn', 143: 'imap2', 161: 'snmp', 162: 'snmp-trap', 163: 'cmip-man',
+    164: 'cmip-agent', 174: 'mailq', 179: 'bgp', 199: 'smux', 209: 'qmtp', 210: 'z3950',
+    345: 'pawserv', 346: 'zserv', 369: 'rpc2portmap', 370: 'codaauth2', 389: 'ldap',
+    427: 'svrloc', 443: 'https', 444: 'snpp', 445: 'microsoft-ds', 464: 'kpasswd',
+    465: 'submissions', 487: 'saft', 512: 'exec', 513: 'login', 514: 'shell', 515: 'printer',
+    538: 'gdomap', 540: 'uucp', 543: 'klogin', 544: 'kshell', 548: 'afpovertcp', 554: 'rtsp',
+    563: 'nntps', 587: 'submission', 607: 'nqs', 628: 'qmqp', 631: 'ipp', 636: 'ldaps',
+    646: 'ldp', 655: 'tinc', 706: 'silc', 749: 'kerberos-adm', 750: 'kerberos4',
+    751: 'kerberos-master', 754: 'krb-prop', 775: 'moira-db', 777: 'moira-update',
+    783: 'spamd', 853: 'domain-s', 871: 'supfilesrv', 873: 'rsync', 989: 'ftps-data',
+    990: 'ftps', 992: 'telnets', 993: 'imaps', 995: 'pop3s', 1080: 'socks', 1093: 'proofd',
+    1094: 'rootd', 1099: 'rmiregistry', 1127: 'supfiledbg', 1178: 'skkserv', 1194: 'openvpn',
+    1236: 'rmtcfg', 1313: 'xtel', 1314: 'xtelw', 1352: 'lotusnote', 1433: 'ms-sql-s',
+    1524: 'ingreslock', 1645: 'datametrics', 1646: 'sa-msg-port', 1649: 'kermit',
+    1677: 'groupwise', 1812: 'radius', 1813: 'radius-acct', 2000: 'cisco-sccp', 2049: 'nfs',
+    2086: 'gnunet', 2101: 'rtcm-sc104', 2119: 'gsigatekeeper', 2121: 'iprop', 2135: 'gris',
+    2401: 'cvspserver', 2430: 'venus', 2431: 'venus-se', 2432: 'codasrv', 2433: 'codasrv-se',
+    2583: 'mon', 2600: 'zebrasrv', 2601: 'zebra',
+    3306: 'mysql', 3389: 'ms-wbt-server', 5432: 'postgresql', 5900: 'vnc',
+    8080: 'http-alt', 8443: 'https-alt'
+}
+
+
+async def scan_ports(ip_address, ports=None, watch_uuid=None, update_signal=None):
+    """
+    Perform fast TCP connect port scan
+
+    Args:
+        ip_address: Target IP address
+        ports: List of ports to scan (default: common service ports)
+        watch_uuid: Optional watch UUID for status updates
+        update_signal: Optional blinker signal for status updates
+
+    Returns:
+        list: List of open ports
+    """
+    if update_signal and watch_uuid:
+        update_signal.send(watch_uuid=watch_uuid, status="Ports")
+
+    if ports is None:
+        # Use ports from /etc/services
+        ports = COMMON_PORTS
+
+    async def check_port(host, port, timeout=0.5):
+        """Fast TCP connect scan for a single port"""
+        try:
+            reader, writer = await asyncio.wait_for(
+                asyncio.open_connection(host, port),
+                timeout=timeout
+            )
+            writer.close()
+            await writer.wait_closed()
+            return port
+        except:
+            return None
+
+    # Scan all ports in parallel
+    scan_tasks = [check_port(ip_address, port) for port in ports]
+    scan_results = await asyncio.gather(*scan_tasks)
+    open_ports = sorted([p for p in scan_results if p is not None])
+
+    return open_ports
+
+
+def format_portscan_results(open_ports):
+    """Format port scan results for output"""
+    lines = []
+    lines.append("=== Port Scan ===")
+
+    if open_ports:
+        lines.append(f"Open Ports ({len(open_ports)} found):")
+        for port in open_ports:
+            service = PORT_SERVICES.get(port, 'unknown')
+            lines.append(f"  {port}: {service}")
+    else:
+        lines.append("No open ports found (or filtered)")
+
+    lines.append("")
+    return '\n'.join(lines)

changedetectionio_osint/steps/registry.py

@@ -0,0 +1,49 @@
+"""
+Step Registry - Automatically discovers and registers all scan steps
+"""
+
+from typing import List, Type
+from .base import ScanStep
+from loguru import logger
+
+
+# Registry of all scan step classes
+_STEPS: List[Type[ScanStep]] = []
+
+
+def register_step(step_class: Type[ScanStep]):
+    """
+    Register a scan step class.
+
+    Args:
+        step_class: ScanStep subclass to register
+    """
+    _STEPS.append(step_class)
+    logger.debug(f"Registered OSINT scan step: {step_class.name} (order: {step_class.order})")
+    return step_class
+
+
+def get_all_steps() -> List[Type[ScanStep]]:
+    """
+    Get all registered scan steps, sorted by order.
+
+    Returns:
+        List of ScanStep classes sorted by their order attribute
+    """
+    return sorted(_STEPS, key=lambda s: s.order)
+
+
+def discover_steps():
+    """
+    Auto-discover and import all step modules.
+
+    This function imports all step modules which triggers their @register_step decorators.
+    """
+    # Import all step modules to trigger registration
+    from . import dns_scan
+    from . import whois_scan
+    from . import http_scan
+    from . import tls_scan
+    from . import port_scan
+    from . import traceroute_scan
+    from . import bgp_scan