osintengine 1.0.2__py3-none-any.whl

src/watson/tools/shodan.py

@@ -0,0 +1,129 @@
+"""Shodan tool — internet-wide infrastructure scanning for OSINT.
+
+Discovers exposed services, industrial control systems, vulnerable
+infrastructure associated with a target. Paid API required.
+"""
+
+from __future__ import annotations
+
+import logging
+
+from .base import OSINTTool
+from .registry import registry
+from ..core.models import Finding, FindingSource
+
+logger = logging.getLogger("watson.shodan")
+
+
+class ShodanTool(OSINTTool):
+    """Internet scanner — discovers exposed services, ICS, vulnerable infra."""
+
+    category = FindingSource.OSINT
+    name = "shodan"
+    description = "Shodan internet scanner — exposed services, industrial control systems, vulnerable infrastructure"
+    free_tier_available = False
+    rate_limit_rps = 1.0
+
+    SHODAN_HOST_SEARCH = "https://api.shodan.io/shodan/host/search"
+
+    async def investigate(self, query: str, context: str = "") -> list[Finding]:
+        """Search Shodan for exposed infrastructure related to the target."""
+        findings: list[Finding] = []
+
+        from watson.api_keys import get_key
+
+        api_key = get_key("shodan")
+        if not api_key:
+            findings.append(self._make_finding(
+                title="🔒 Shodan API key not configured",
+                description=(
+                    "Shodan provides internet-wide infrastructure scanning — exposed "
+                    "services, industrial control systems, vulnerable ports. "
+                    "Install your API key in Settings → API Vault to enable.\n\n"
+                    "Get a key: https://account.shodan.io/"
+                ),
+                confidence=0.0,
+                source_url="https://account.shodan.io/",
+            ))
+            return findings
+
+        # Extract searchable terms
+        search_term = self._extract_search_term(query)
+        if not search_term:
+            return findings
+
+        try:
+            import httpx
+
+            async with httpx.AsyncClient(timeout=20) as client:
+                resp = await client.get(
+                    f"{self.SHODAN_HOST_SEARCH}?key={api_key}&query={search_term}&facets=org,os,port",
+                )
+                resp.raise_for_status()
+                data = resp.json()
+
+                matches = data.get("matches", [])
+                total = data.get("total", 0)
+
+                if matches:
+                    entries = []
+                    for m in matches[:8]:
+                        ip = m.get("ip_str", "?")
+                        port = m.get("port", "?")
+                        org = m.get("org", "?")
+                        os_name = m.get("os", "?") or "?"
+                        hostnames = ", ".join(m.get("hostnames", [])[:2]) or "none"
+                        product = m.get("product", m.get("_shodan", {}).get("module", "unknown"))
+
+                        entries.append(
+                            f"- **{ip}:{port}** — {product} ({org})\n"
+                            f"  OS: {os_name} | Hostnames: {hostnames}"
+                        )
+
+                    findings.append(self._make_finding(
+                        title=f"🌐 Shodan: {len(matches)} exposed services for '{search_term}' ({total} total)",
+                        description="\n".join(entries[:6]),
+                        confidence=0.85,
+                        evidence=[f"https://www.shodan.io/search?query={search_term}"],
+                        total_results=total,
+                    ))
+                else:
+                    findings.append(self._make_finding(
+                        title=f"🌐 Shodan: No exposed services found for '{search_term}'",
+                        description="No internet-facing infrastructure matching the query was discovered.",
+                        confidence=0.6,
+                    ))
+
+        except Exception as e:
+            logger.warning("shodan_search_failed: %s", e)
+            findings.append(self._make_finding(
+                title=f"⚠ Shodan search failed: {str(e)[:100]}",
+                description="API call failed. Check your key or try again later.",
+                confidence=0.0,
+            ))
+
+        return findings
+
+    @staticmethod
+    def _extract_search_term(query: str) -> str:
+        """Extract domain, IP, or org name for Shodan search."""
+        import re
+
+        # IP address
+        ip_match = re.search(r'\b(?:\d{1,3}\.){3}\d{1,3}\b', query)
+        if ip_match:
+            return ip_match.group(0)
+
+        # Domain
+        domain_match = re.search(r'(?:org|domain|hostname)\s*[:=]?\s*([a-zA-Z0-9][-a-zA-Z0-9]*\.[a-zA-Z]{2,})', query, re.IGNORECASE)
+        if domain_match:
+            return domain_match.group(1)
+
+        # Use first 3 words as org search
+        words = query.split()[:3]
+        return f'org:"{" ".join(words)}"'
+
+
+# Register
+shodan_tool = ShodanTool()
+registry.register(shodan_tool)

src/watson/tools/social_media.py

@@ -0,0 +1,160 @@
+"""Social Media tool — cross-platform profile discovery and analysis."""
+
+from __future__ import annotations
+
+import asyncio
+
+from .base import OSINTTool
+from .registry import registry
+from ..core.models import Finding, FindingSource
+from ..utils.http import get_client
+from ..utils.helpers import clean_username
+
+
+# Common social media platforms and their profile URL patterns
+SOCIAL_PLATFORMS = [
+    ("Twitter/X", "https://x.com/{username}"),
+    ("Instagram", "https://instagram.com/{username}"),
+    ("LinkedIn", "https://linkedin.com/in/{username}"),
+    ("GitHub", "https://github.com/{username}"),
+    ("Reddit", "https://reddit.com/user/{username}"),
+    ("TikTok", "https://tiktok.com/@{username}"),
+    ("YouTube", "https://youtube.com/@{username}"),
+    ("Facebook", "https://facebook.com/{username}"),
+    ("Telegram", "https://t.me/{username}"),
+    ("Medium", "https://medium.com/@{username}"),
+    ("Substack", "https://{username}.substack.com"),
+    ("Pinterest", "https://pinterest.com/{username}"),
+    ("Twitch", "https://twitch.tv/{username}"),
+    ("Snapchat", "https://snapchat.com/add/{username}"),
+]
+
+
+class SocialMediaTool(OSINTTool):
+    """Discover social media profiles across platforms by username."""
+
+    category = FindingSource.SOCIAL_MEDIA
+    name = "social-media"
+    description = "Cross-platform profile discovery, username search, social presence mapping"
+    free_tier_available = True
+    rate_limit_rps = 2.0
+
+    async def investigate(self, query: str, context: str = "") -> list[Finding]:
+        findings: list[Finding] = []
+
+        # Extract potential usernames
+        usernames = self._extract_usernames(query)
+        if not usernames:
+            return findings
+
+        client = get_client(rate_limit=self.rate_limit_rps)
+
+        for username in usernames[:3]:  # Max 3 usernames
+            username_clean = clean_username(username)
+
+            # Check profile URLs in parallel
+            platforms_found: list[str] = []
+            tasks = []
+
+            for platform_name, url_template in SOCIAL_PLATFORMS:
+                url = url_template.format(username=username_clean)
+                tasks.append(self._check_profile(client, platform_name, url))
+
+            results = await asyncio.gather(*tasks)
+
+            for name, url, exists in results:
+                if exists:
+                    platforms_found.append(f"[{name}]({url})")
+
+            if platforms_found:
+                findings.append(
+                    self._make_finding(
+                        title=f"👤 Social profiles for '{username_clean}'",
+                        description=(
+                            f"Found {len(platforms_found)} profiles across platforms:\n"
+                            + "\n".join(f"- {p}" for p in platforms_found)
+                        ),
+                        confidence=0.85 if len(platforms_found) >= 3 else 0.5,
+                        username=username_clean,
+                        platform_count=len(platforms_found),
+                        platforms=platforms_found,
+                    )
+                )
+            else:
+                findings.append(
+                    self._make_finding(
+                        title=f"No public profiles found for '{username_clean}'",
+                        description=(
+                            f"Checked {len(SOCIAL_PLATFORMS)} platforms. "
+                            "The username may not exist, be private, or use a different handle."
+                        ),
+                        confidence=0.3,
+                        username=username_clean,
+                    )
+                )
+
+            # Generate a profile URL list regardless (for manual checking)
+            all_urls = [
+                f"[{name}]({url.format(username=username_clean)})"
+                for name, url in SOCIAL_PLATFORMS[:8]
+            ]
+            findings.append(
+                self._make_finding(
+                    title=f"🔗 Profile links for '{username_clean}' (manual check)",
+                    description="Quick links to check manually:\n" + "\n".join(f"- {u}" for u in all_urls),
+                    confidence=0.6,
+                    username=username_clean,
+                )
+            )
+
+        return findings
+
+    async def _check_profile(
+        self, client, platform_name: str, url: str
+    ) -> tuple[str, str, bool]:
+        """Check if a social media profile exists. Returns (name, url, exists)."""
+        try:
+            response = await client.get(url)
+            # A 200 means the profile exists and is public
+            # Some platforms return 200 with a "not found" page, but this is a good heuristic
+            return platform_name, url, True
+        except Exception:
+            return platform_name, url, False
+
+    def _extract_usernames(self, text: str) -> list[str]:
+        """Extract potential usernames from query text."""
+        usernames = []
+
+        # Look for @handles
+        import re
+
+        handles = re.findall(r"@(\w{3,30})", text)
+        usernames.extend(handles)
+
+        # Look for "username X" or "handle X" patterns
+        patterns = [
+            r"(?:username|handle|profile|account)\s+(?:is\s+)?['\"]?(\w{3,30})['\"]?",
+            r"(?:find|search|look\s+up|check)\s+(?:the\s+)?(?:social\s+)?(?:media\s+)?(?:profile|account)(?:\s+for)?\s+['\"]?(\w{3,30})['\"]?",
+        ]
+        for pattern in patterns:
+            match = re.search(pattern, text, re.IGNORECASE)
+            if match:
+                usernames.append(match.group(1))
+
+        # Fallback: treat any single CamelCase word as a potential username
+        if not usernames:
+            word_match = re.search(r'\b([A-Za-z][A-Za-z0-9_]{2,30})\b', text)
+            if word_match:
+                word = word_match.group(1)
+                # Don't use common investigative words as usernames
+                if word.lower() not in ("who", "what", "where", "when", "why", "how",
+                    "the", "and", "for", "with", "company", "person", "domain",
+                    "investigate", "research", "search", "find", "look", "check"):
+                    usernames.append(word)
+
+        return list(dict.fromkeys(usernames))  # Deduplicated
+
+
+# Register
+social_media_tool = SocialMediaTool()
+registry.register(social_media_tool)

src/watson/tools/websites.py

@@ -0,0 +1,291 @@
+"""Websites & Domains tool — WHOIS, Wayback Machine, DNS, SSL certificates."""
+
+from __future__ import annotations
+
+import asyncio
+from datetime import datetime
+
+from .base import OSINTTool
+from .registry import registry
+from ..core.models import Finding, FindingSeverity, FindingSource
+from ..utils.http import get_client
+from ..utils.helpers import extract_domain
+
+
+class WebsitesTool(OSINTTool):
+    """Investigate domains — WHOIS, Internet Archive, SSL certificates, DNS records."""
+
+    category = FindingSource.WEBSITES
+    name = "websites-domains"
+    description = "WHOIS lookup, Wayback Machine history, SSL certificates (crt.sh), subdomain discovery"
+    free_tier_available = True
+    rate_limit_rps = 3.0
+
+    WAYBACK_CDX = "https://web.archive.org/cdx/search/cdx"
+    CRTSH_API = "https://crt.sh/"
+    DNS_OVER_HTTPS = "https://dns.google/resolve"
+
+    async def investigate(self, query: str, context: str = "") -> list[Finding]:
+        findings: list[Finding] = []
+
+        domains = self._extract_domains(query)
+        if not domains:
+            return findings
+
+        import httpx
+        for domain in domains[:3]:
+            clean = extract_domain(domain)
+
+            async def _wayback():
+                try:
+                    params = {"url": f"*.{clean}/*", "output": "json", "limit": 5, "fl": "timestamp,original"}
+                    async with httpx.AsyncClient(timeout=10) as c:
+                        resp = await c.get(self.WAYBACK_CDX, params=params)
+                        if resp.status_code == 200:
+                            snapshots = resp.json()
+                            if isinstance(snapshots, list) and snapshots:
+                                first = snapshots[0][0] if snapshots[0] else "unknown"
+                                last = snapshots[-1][0] if snapshots[-1] else "unknown"
+                                return self._make_finding(
+                                    title=f"📚 Wayback Machine: {clean}",
+                                    description=f"First archived: {first[:4]}-{first[4:6]}-{first[6:8]}. "
+                                    f"Latest snapshot: {last[:4]}-{last[4:6]}-{last[6:8]}. "
+                                    f"Total unique snapshots in recent window: {len(snapshots)}.",
+                                    evidence=[f"https://web.archive.org/web/*/{clean}"],
+                                    confidence=0.95,
+                                    domain=clean,
+                                )
+                except Exception:
+                    pass
+                return None
+
+            async def _crtsh():
+                try:
+                    url = f"{self.CRTSH_API}?q=%25.{clean}&output=json"
+                    async with httpx.AsyncClient(timeout=15) as c:
+                        resp = await c.get(url, headers={"User-Agent": "WatsonOSINT/0.3"})
+                        if resp.status_code == 200:
+                            certs = resp.json()
+                            if isinstance(certs, list) and certs:
+                                subdomains = set()
+                                for cert in certs[:50]:
+                                    names = cert.get("name_value", "")
+                                    for n in names.split("\\n"):
+                                        n = n.strip()
+                                        if n and n != clean and not n.startswith("*."):
+                                            subdomains.add(n)
+                                return self._make_finding(
+                                    title=f"🔐 SSL certificates: {clean}",
+                                    description=f"{len(certs)} certificates found. "
+                                    f"{len(subdomains)} unique subdomains: {', '.join(sorted(list(subdomains))[:10])}",
+                                    evidence=[f"https://crt.sh/?q=%.{clean}"],
+                                    confidence=0.85,
+                                    domain=clean,
+                                )
+                        elif resp.status_code >= 400:
+                            return self._make_finding(
+                                title=f"⚠️ crt.sh lookup failed for {clean}",
+                                description=f"SSL certificate lookup error: HTTP {resp.status_code}. "
+                                f"Try manually: https://crt.sh/?q=%.{clean}",
+                                evidence=[f"https://crt.sh/?q=%.{clean}"],
+                                confidence=0.1,
+                                domain=clean,
+                            )
+                except Exception:
+                    pass
+                return None
+
+            async def _dns():
+                try:
+                    async with httpx.AsyncClient(timeout=10) as c:
+                        a_resp = await c.get(f"{self.DNS_OVER_HTTPS}?name={clean}&type=A")
+                        mx_resp = await c.get(f"{self.DNS_OVER_HTTPS}?name={clean}&type=MX")
+                        ns_resp = await c.get(f"{self.DNS_OVER_HTTPS}?name={clean}&type=NS")
+                        txt_resp = await c.get(f"{self.DNS_OVER_HTTPS}?name={clean}&type=TXT")
+                        a_data = a_resp.json() if a_resp.status_code == 200 else {}
+                        mx_data = mx_resp.json() if mx_resp.status_code == 200 else {}
+                        ns_data = ns_resp.json() if ns_resp.status_code == 200 else {}
+                        txt_data = txt_resp.json() if txt_resp.status_code == 200 else {}
+                        a_records = [a.get("data", "") for a in a_data.get("Answer", [])]
+                        mx_records = [m.get("data", "") for m in mx_data.get("Answer", [])]
+                        ns_records = [n.get("data", "") for n in ns_data.get("Answer", [])]
+                        txt_records = [t.get("data", "") for t in txt_data.get("Answer", [])]
+                        if a_records or mx_records:
+                            return self._make_finding(
+                                title=f"🌐 DNS records for {clean}",
+                                description=f"- A: {', '.join(a_records[:3])} "
+                                f"- MX: {', '.join(mx_records[:3])} "
+                                f"- NS: {', '.join(ns_records[:3])} "
+                                f"- TXT: {', '.join(txt_records[:3])}",
+                                evidence=[f"https://dns.google/resolve?name={clean}"],
+                                confidence=0.9,
+                                domain=clean,
+                            )
+                except Exception:
+                    pass
+                return None
+
+            wayback, crt, dns = await asyncio.gather(
+                _wayback(), _crtsh(), _dns(),
+            )
+            if wayback: findings.append(wayback)
+            if crt: findings.append(crt)
+            if dns: findings.append(dns)
+
+        return findings
+
+    async def _check_wayback(self, client, domain: str) -> Finding | None:
+        """Check Internet Archive Wayback Machine for domain history."""
+        try:
+            params = {
+                "url": f"*.{domain}/*",
+                "output": "json",
+                "limit": 5,
+                "fl": "timestamp,original,statuscode",
+                "collapse": "digest",
+            }
+            data = await client.get_json(self.WAYBACK_CDX, params=params)
+
+            if isinstance(data, list) and len(data) > 1:
+                # Skip header row (first element is column names)
+                rows = data[1:] if isinstance(data[0], list) and data[0][0] == "timestamp" else data
+                if not rows:
+                    return None
+                first = rows[-1]
+                latest = rows[0]
+
+                first_date = datetime.strptime(first[0][:8], "%Y%m%d").strftime("%b %d, %Y")
+                latest_date = datetime.strptime(latest[0][:8], "%Y%m%d").strftime("%b %d, %Y")
+
+                return self._make_finding(
+                    title=f"📚 Wayback Machine: {domain}",
+                    description=(
+                        f"First archived: {first_date}. "
+                        f"Latest snapshot: {latest_date}. "
+                        f"Total unique snapshots in recent window: {len(data)}."
+                    ),
+                    evidence=[
+                        f"https://web.archive.org/web/*/{domain}",
+                        f"https://web.archive.org/web/{latest[0]}/{latest[1]}",
+                    ],
+                    confidence=0.95,
+                    domain=domain,
+                    first_snapshot=first[0],
+                    latest_snapshot=latest[0],
+                )
+        except Exception as e:
+            return self._make_finding(
+                title=f"⚠️ Wayback Machine unavailable for {domain}",
+                description=f"Could not retrieve archive data: {str(e)[:200]}",
+                confidence=0.0,
+                severity=FindingSeverity.LOW,
+            )
+        return None
+
+    async def _check_crtsh(self, client, domain: str) -> Finding | None:
+        """Check SSL certificate transparency logs via crt.sh."""
+        try:
+            url = f"{self.CRTSH_API}?q=%25.{domain}&output=json"
+            data = await client.get_json(url)
+
+            if isinstance(data, list) and data:
+                # Extract unique subdomains
+                subdomains: set[str] = set()
+                for entry in data[:50]:
+                    names = entry.get("name_value", "").split("\n")
+                    for name in names:
+                        name = name.strip().lstrip("*.")
+                        if name and domain in name:
+                            subdomains.add(name)
+
+                subdomain_list = sorted(subdomains)[:10]
+
+                return self._make_finding(
+                    title=f"🔒 SSL certs: {len(subdomains)} subdomains found for {domain}",
+                    description=(
+                        f"Discovered {len(subdomains)} unique names via certificate transparency. "
+                        f"First 10:\n" + "\n".join(f"- `{s}`" for s in subdomain_list)
+                    ),
+                    evidence=[f"https://crt.sh/?q=%.{domain}"],
+                    confidence=0.9,
+                    domain=domain,
+                    subdomain_count=len(subdomains),
+                )
+        except Exception as e:
+            return self._make_finding(
+                title=f"⚠️ crt.sh lookup failed for {domain}",
+                description=f"SSL certificate lookup error: {str(e)[:200]}. Try manually: https://crt.sh/?q=%.{domain}",
+                evidence=[f"https://crt.sh/?q=%.{domain}"],
+                confidence=0.1,
+                severity=FindingSeverity.LOW,
+            )
+        return None
+
+    async def _check_dns(self, client, domain: str) -> Finding | None:
+        """Check DNS records via Google DNS-over-HTTPS."""
+        try:
+            async def _query(rt: str) -> str | None:
+                try:
+                    data = await client.get_json(
+                        self.DNS_OVER_HTTPS, params={"name": domain, "type": rt}
+                    )
+                    answers = data.get("Answer", [])
+                    if answers:
+                        values = [a["data"] for a in answers[:3]]
+                        return f"{rt}: {', '.join(values)}"
+                except Exception:
+                    pass
+                return None
+
+            results = await asyncio.gather(
+                *[_query(rt) for rt in ["A", "AAAA", "MX", "NS", "TXT"]]
+            )
+            records_found = [r for r in results if r]
+
+            if records_found:
+                return self._make_finding(
+                    title=f"🌐 DNS records for {domain}",
+                    description="\n".join(f"- {r}" for r in records_found),
+                    confidence=0.9,
+                    domain=domain,
+                )
+            else:
+                return self._make_finding(
+                    title=f"⚠️ No DNS records found for {domain}",
+                    description="DNS-over-HTTPS returned no records. Domain may not resolve or is parked.",
+                    confidence=0.3,
+                    severity=FindingSeverity.LOW,
+                )
+        except Exception as e:
+            return self._make_finding(
+                title=f"⚠️ DNS lookup failed for {domain}",
+                description=f"DNS-over-HTTPS error: {str(e)[:200]}",
+                confidence=0.0,
+                severity=FindingSeverity.LOW,
+            )
+
+    def _extract_domains(self, text: str) -> list[str]:
+        """Extract domain names from text. Falls back to deriving domain from single-word queries."""
+        import re
+
+        pattern = r"(?:https?://)?(?:www\.)?([a-zA-Z0-9][-a-zA-Z0-9]*\.[a-zA-Z]{2,}(?:\.[a-zA-Z]{2,})?)"
+        matches = re.findall(pattern, text)
+        domains = list(dict.fromkeys(matches))
+
+        # If no domain found and query looks like a company/product name, derive it
+        if not domains:
+            # Check for single capitalized word or CamelCase (e.g. "OpenAI", "DeepSeek")
+            word_match = re.search(r'\b([A-Za-z][A-Za-z0-9]{2,}(?:\.[a-z]{2,})?)\b', text)
+            if word_match:
+                word = word_match.group(1).lower()
+                if '.' not in word:
+                    domains = [f"{word}.com", f"{word}.org", f"{word}.io"]
+                else:
+                    domains = [word]
+
+        return domains
+
+
+# Register
+websites_tool = WebsitesTool()
+registry.register(websites_tool)