backhoe-osint 0.5.0__py3-none-any.whl

backhoe/backends/censys.py

@@ -0,0 +1,198 @@
+"""
+Censys backend — a second richer-port-data source for infra-check,
+alongside Shodan (backends/shodan.py), whose shape this module
+deliberately mirrors: same raw dict field names (port/service/product/
+version/vulns), same source-key-first exception hierarchy, same
+KeyProvider registration pattern. That's what lets scoring.py and
+report.py handle Censys findings with zero changes — they already
+handle this shape generically, built for and validated against Shodan.
+
+Needs an API key: backhoe.keys.get_api_key(CENSYS_PROVIDER) resolves one
+(CENSYS_API_KEY env var, then a locally stored key, then an interactive
+prompt — see keys.py). The "key" here is a Censys Personal Access Token
+(PAT), used as a Bearer token, not the older API-ID+secret pair Censys's
+now-deprecated v2 Search API used — confirmed live against Censys's
+current docs (docs.censys.com) during development: their own docs say
+"all scripted access should use" the newer Platform API (v3) this module
+targets, not the legacy one. This is a real API generation Shodan-style
+memory-only development couldn't have caught.
+
+validate_key() hits /v3/accounts/users/credits, confirmed live in
+Censys's docs — quoted twice — to cost no credits, letting keys.py
+re-validate a stored token on every call without burning quota (same
+role as Shodan's /api-info).
+
+What's NOT confirmed live (docs summarization truncated the raw OpenAPI
+schema before the exact response shape came through): the precise
+404/no-data behavior for the host-lookup endpoint (treated as an empty
+result, matching Shodan's precedent and general REST convention), and
+the exact shape of a service's `vulns` field. The clearest signal found
+says it's a list of {"id": "CVE-...", ...} objects — different from
+Shodan's plain-string-or-dict shape — so _extract_vuln_ids() below
+handles THREE possible shapes rather than assuming one.
+"""
+from __future__ import annotations
+
+import requests
+
+from ..keys import KeyProvider, KeyValidationError
+from ..schema import Finding, FindingType
+
+BASE_URL = "https://api.platform.censys.io/v3"
+DEFAULT_TIMEOUT = 10
+
+
+class CensysError(Exception):
+    """A real failure calling Censys — never raised for "no data for this
+    host", which is a legitimate empty result (see lookup_host)."""
+
+
+class CensysAPIError(CensysError):
+    """lookup_host() failed for a reason other than "not indexed": bad
+    token, rate limited, Censys's own server error, network failure, or
+    an unparseable response."""
+
+
+class CensysValidationError(CensysError, KeyValidationError):
+    """validate_key() itself failed inconclusively (network error, 5xx,
+    unparseable response) — NOT the same as the key being confirmed
+    wrong. keys.py treats this as "can't confirm, use the key anyway",
+    not as a reason to reprompt."""
+
+
+def _redact_key(text: str, key: str) -> str:
+    """Defensive belt-and-suspenders: Censys's Bearer-token auth means the
+    key never appears in a request URL/query string the way Shodan's did,
+    but some requests error paths (e.g. InvalidHeader on a key containing
+    a stray control character) embed repr(key) in their message instead
+    of the raw string — the escaped form (\\n, \\r, etc.) never matches a
+    literal .replace() against the raw key, so redact both forms."""
+    if not key:
+        return text
+    text = text.replace(key, "<redacted>")
+    escaped = repr(key)[1:-1]
+    if escaped and escaped != key:
+        text = text.replace(escaped, "<redacted>")
+    return text
+
+
+def validate_key(key: str) -> bool:
+    try:
+        resp = requests.get(
+            f"{BASE_URL}/accounts/users/credits",
+            headers={"Authorization": f"Bearer {key}"},
+            timeout=DEFAULT_TIMEOUT,
+        )
+    except requests.RequestException as exc:
+        raise CensysValidationError(f"network error contacting Censys: {_redact_key(str(exc), key)}") from exc
+
+    if resp.status_code == 200:
+        return True
+    if resp.status_code == 401:
+        return False
+    raise CensysValidationError(f"unexpected Censys /accounts/users/credits status {resp.status_code}")
+
+
+def lookup_host(ip: str, key: str) -> list[Finding]:
+    try:
+        resp = requests.get(
+            f"{BASE_URL}/global/asset/host/{ip}",
+            headers={"Authorization": f"Bearer {key}"},
+            timeout=DEFAULT_TIMEOUT,
+        )
+    except requests.RequestException as exc:
+        raise CensysAPIError(f"network error contacting Censys: {_redact_key(str(exc), key)}") from exc
+
+    if resp.status_code == 404:
+        return []
+    if resp.status_code != 200:
+        raise CensysAPIError(f"Censys host lookup returned status {resp.status_code}")
+
+    try:
+        data = resp.json()
+    except ValueError as exc:
+        raise CensysAPIError(f"unparseable JSON from Censys: {exc}") from exc
+
+    return _to_findings(ip, data)
+
+
+def _extract_vuln_ids(entry: dict) -> list[str]:
+    """CVE ids for one service entry. Shape not confirmed live (see module
+    docstring) — handle a dict keyed by CVE id, a plain list of CVE-id
+    strings, and a list of {"id": ...} objects, rather than assuming one."""
+    vulns = entry.get("vulns")
+    if not vulns:
+        return []
+    if isinstance(vulns, dict):
+        return sorted(vulns.keys())
+    if isinstance(vulns, list):
+        ids: list[str] = []
+        for v in vulns:
+            if isinstance(v, dict):
+                vid = v.get("id")
+                if vid:
+                    ids.append(str(vid))
+            elif v:
+                ids.append(str(v))
+        return sorted(ids)
+    return []
+
+
+def _to_findings(ip: str, data: dict) -> list[Finding]:
+    # Group by port before building Finding objects, same reason as
+    # shodan.py's _to_findings: two service entries for the same port
+    # would otherwise share a dedup key AND source ("censys"), letting
+    # merge_findings() silently let the second overwrite the first's raw
+    # payload (the exact Critical bug Shodan's final review caught).
+    result = data.get("result")
+    if not isinstance(result, dict):
+        raise CensysAPIError("unexpected Censys response shape: no 'result' object in body")
+    resource = result.get("resource")
+    if not isinstance(resource, dict):
+        raise CensysAPIError("unexpected Censys response shape: no 'resource' object in body")
+
+    by_port: dict[int, dict] = {}
+    try:
+        for svc in resource.get("services", []):
+            port = svc.get("port")
+            if port is None:
+                continue
+            try:
+                port = int(port)
+            except (TypeError, ValueError):
+                continue
+            merged = by_port.setdefault(
+                port, {"service": None, "product": None, "version": None, "vulns": []}
+            )
+            protocol = svc.get("protocol")
+            if merged["service"] is None and protocol is not None:
+                merged["service"] = protocol
+            for sw in svc.get("software") or []:
+                if merged["product"] is None and sw.get("product") is not None:
+                    merged["product"] = sw.get("product")
+                    merged["version"] = sw.get("version")
+            for v in _extract_vuln_ids(svc):
+                if v not in merged["vulns"]:
+                    merged["vulns"].append(v)
+    except (AttributeError, TypeError) as exc:
+        raise CensysAPIError(f"unexpected Censys response shape: {exc}") from exc
+
+    findings: list[Finding] = []
+    for port, payload in by_port.items():
+        findings.append(
+            Finding(
+                type=FindingType.OPEN_PORT,
+                value=f"{ip}:{port}",
+                source="censys",
+                raw={"port": port, **payload},
+            )
+        )
+    return findings
+
+
+CENSYS_PROVIDER = KeyProvider(
+    name="censys",
+    env_var="CENSYS_API_KEY",
+    prompt_label="Censys API key (Personal Access Token)",
+    validate=validate_key,
+)

backhoe/backends/crtsh.py

@@ -0,0 +1,87 @@
+"""
+crt.sh backend — certificate transparency log lookups.
+
+Chosen as BACKHOE's first backend because it needs zero API keys and
+zero rate-limit headaches, so `backhoe domain-audit <domain>` works
+out of the box with no setup wizard required first.
+"""
+
+import requests
+from datetime import datetime
+
+from ..schema import Finding, FindingType
+
+CRTSH_URL = "https://crt.sh/"
+
+
+class CrtShError(Exception):
+    """Raised when crt.sh can't be queried or returns something unusable.
+
+    This is a hard failure, not a Finding — a lookup that failed has zero
+    subdomains in it, and rendering that as a fake "finding" with
+    confidence 0.0 (the old behavior) buries a real error inside the
+    results table instead of telling the user their scan didn't run.
+    """
+
+
+def run(domain: str) -> list[Finding]:
+    """
+    Query crt.sh for every certificate issued for *.{domain} and pull
+    out the unique subdomains mentioned in them.
+
+    Raises CrtShError on any failure — callers decide how to surface that,
+    but it must never be silently swallowed into an empty-looking result.
+    """
+    try:
+        resp = requests.get(
+            CRTSH_URL,
+            params={"q": f"%.{domain}", "output": "json"},
+            timeout=30,
+            headers={"User-Agent": "backhoe-osint/0.1"},
+        )
+        resp.raise_for_status()
+        records = resp.json()
+    except requests.RequestException as e:
+        raise CrtShError(f"crt.sh request failed: {e}") from e
+    except ValueError as e:
+        raise CrtShError(f"crt.sh returned unparseable data: {e}") from e
+
+    if not isinstance(records, list):
+        raise CrtShError(f"crt.sh returned an unexpected response shape: {type(records).__name__}")
+
+    findings: list[Finding] = []
+    seen_subdomains: dict[str, datetime | None] = {}
+
+    for rec in records:
+        name_value = rec.get("name_value", "")
+        not_before = rec.get("not_before")
+        parsed_date = None
+        if not_before:
+            try:
+                parsed_date = datetime.strptime(not_before, "%Y-%m-%dT%H:%M:%S")
+            except ValueError:
+                pass
+
+        # a single cert can list multiple names (SANs) separated by newlines
+        for name in name_value.split("\n"):
+            name = name.strip().lower()
+            if not name or "*" in name:
+                continue
+            # keep the most recent issue date we've seen for this name
+            if name not in seen_subdomains or (
+                parsed_date and (seen_subdomains[name] is None or parsed_date > seen_subdomains[name])
+            ):
+                seen_subdomains[name] = parsed_date
+
+    for name, issued in seen_subdomains.items():
+        findings.append(
+            Finding(
+                type=FindingType.SUBDOMAIN,
+                value=name,
+                source="crt.sh",
+                first_seen=issued,
+                raw={"domain": domain},
+            )
+        )
+
+    return findings

backhoe/backends/dns_checks.py

@@ -0,0 +1,97 @@
+"""
+DNS-based recon checks — MX/SPF/DMARC (email security posture) and PTR
+(reverse DNS). All genuinely verifiable with no API key: DNS just answers
+with what's actually published. The only real engineering problem is
+telling "this record doesn't exist" (a legitimate finding) apart from
+"we couldn't ask" (a real failure) — conflating them is exactly the kind
+of silent-failure behavior that makes OSINT tools' results untrustworthy.
+"""
+from __future__ import annotations
+
+import socket
+
+import dns.exception
+import dns.resolver
+
+
+class DnsCheckError(Exception):
+    """A real DNS infrastructure failure (timeout, no nameservers, the
+    queried domain doesn't exist at all). Never raised for an authoritative
+    'this specific record doesn't exist' answer — that's a finding, not
+    an error.
+    """
+
+
+def resolve_a_records(domain: str) -> list[str]:
+    try:
+        _, _, ips = socket.gethostbyname_ex(domain)
+        return ips
+    except socket.gaierror as e:
+        raise DnsCheckError(f"could not resolve {domain}: {e}") from e
+
+
+def reverse_dns(ip: str) -> str | None:
+    try:
+        hostname, _, _ = socket.gethostbyaddr(ip)
+        return hostname
+    except socket.herror:
+        return None  # no PTR record — a legitimate, common outcome
+    except socket.gaierror as e:
+        raise DnsCheckError(f"reverse DNS lookup for {ip} failed: {e}") from e
+
+
+def check_mx(domain: str) -> list[str]:
+    try:
+        answers = dns.resolver.resolve(domain, "MX")
+        return sorted(str(r.exchange).rstrip(".") for r in answers)
+    except dns.resolver.NoAnswer:
+        return []
+    except dns.resolver.NXDOMAIN as e:
+        raise DnsCheckError(f"{domain} does not exist (NXDOMAIN)") from e
+    except (dns.exception.Timeout, dns.resolver.NoNameservers) as e:
+        raise DnsCheckError(f"MX lookup for {domain} failed: {e}") from e
+
+
+def check_spf(domain: str) -> str | None:
+    try:
+        answers = dns.resolver.resolve(domain, "TXT")
+    except dns.resolver.NoAnswer:
+        return None
+    except dns.resolver.NXDOMAIN as e:
+        raise DnsCheckError(f"{domain} does not exist (NXDOMAIN)") from e
+    except (dns.exception.Timeout, dns.resolver.NoNameservers) as e:
+        raise DnsCheckError(f"TXT lookup for {domain} failed: {e}") from e
+
+    for record in answers:
+        txt = _join_txt(record)
+        if txt.lower().startswith("v=spf1"):
+            return txt
+    return None
+
+
+def check_dmarc(domain: str) -> tuple[bool, str | None]:
+    """Returns (present, policy). policy is the p= value (none/quarantine/reject)."""
+    try:
+        answers = dns.resolver.resolve(f"_dmarc.{domain}", "TXT")
+    except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN):
+        return False, None
+    except (dns.exception.Timeout, dns.resolver.NoNameservers) as e:
+        raise DnsCheckError(f"DMARC lookup for {domain} failed: {e}") from e
+
+    for record in answers:
+        txt = _join_txt(record)
+        if txt.lower().startswith("v=dmarc1"):
+            policy = None
+            for part in txt.split(";"):
+                part = part.strip()
+                if part.lower().startswith("p="):
+                    policy = part.split("=", 1)[1].strip().lower()
+            return True, policy
+    return False, None
+
+
+def _join_txt(record) -> str:
+    # A TXT record's value can be split across multiple quoted strings.
+    return "".join(
+        part.decode() if isinstance(part, bytes) else part for part in record.strings
+    )

backhoe/backends/gravatar.py

@@ -0,0 +1,33 @@
+"""
+Gravatar existence check — a long-standing, well-known OSINT technique:
+Gravatar exposes whether a public profile exists for an email's hash with
+no authentication required. This is only ever a positive signal (a live,
+human-managed inbox with a public profile), never a breach or exposure
+by itself.
+"""
+from __future__ import annotations
+
+import hashlib
+
+import requests
+
+GRAVATAR_URL = "https://www.gravatar.com/avatar/{h}?d=404"
+
+
+class GravatarError(Exception):
+    """Raised on a network failure — never for a legitimate 404 (no profile)."""
+
+
+def check_gravatar(email: str, *, timeout: float = 10.0) -> bool:
+    digest = hashlib.sha256(email.strip().lower().encode()).hexdigest()
+    url = GRAVATAR_URL.format(h=digest)
+    try:
+        resp = requests.get(url, timeout=timeout, headers={"User-Agent": "backhoe/0.1"})
+    except requests.RequestException as e:
+        raise GravatarError(f"Gravatar check failed: {e}") from e
+
+    if resp.status_code == 200:
+        return True
+    if resp.status_code == 404:
+        return False
+    raise GravatarError(f"Gravatar returned unexpected status {resp.status_code}")

backhoe/backends/netcheck.py

@@ -0,0 +1,33 @@
+"""
+Detects whether outbound TCP/TLS traffic is being transparently
+intercepted (a corporate proxy, security sandbox, some VPNs). Port scans
+and raw TLS-certificate fetches are worthless under interception: every
+port looks "open" and every certificate is the interceptor's re-signed
+stand-in, not the real target's — not a hypothetical, this is exactly
+what happened testing against a real domain in a sandboxed environment
+during development (the "certificate" came back issued by the sandbox's
+own egress gateway, not any real CA).
+
+Detected by handshaking against a guaranteed-unassigned IP (TEST-NET-3,
+RFC 5737) — a real network can never route there, so a completed
+handshake proves something is answering on its behalf.
+"""
+from __future__ import annotations
+
+import socket
+import ssl
+
+_CANARY_IP = "203.0.113.1"  # RFC 5737 TEST-NET-3 — reserved, never routed
+_CANARY_PORT = 443
+
+
+def tcp_tls_is_intercepted(*, timeout: float = 5.0) -> bool:
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+    try:
+        with socket.create_connection((_CANARY_IP, _CANARY_PORT), timeout=timeout) as sock:
+            with ctx.wrap_socket(sock):
+                return True
+    except OSError:
+        return False

backhoe/backends/portscan.py

@@ -0,0 +1,52 @@
+"""
+Lightweight TCP connect scan — a small, fixed set of commonly-exposed
+ports, not a full sweep. This is active scanning, not passive OSINT:
+only ever run it against infrastructure you own or are explicitly
+authorized to test.
+
+Note: from behind a network that transparently intercepts outbound
+connections (many corporate networks, CI sandboxes, some VPNs), every
+port will falsely appear "open" because the connection succeeds against
+the interceptor, not the real host. Trust these results only when run
+from a direct, unproxied network.
+"""
+from __future__ import annotations
+
+import socket
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+COMMON_PORTS: dict[int, str] = {
+    21: "ftp",
+    22: "ssh",
+    25: "smtp",
+    80: "http",
+    443: "https",
+    3306: "mysql",
+    3389: "rdp",
+    8080: "http-alt",
+    8443: "https-alt",
+}
+
+
+def scan_ports(
+    host: str,
+    ports: dict[int, str] | None = None,
+    *,
+    timeout: float = 3.0,
+    workers: int = 10,
+) -> dict[int, bool]:
+    ports = ports if ports is not None else COMMON_PORTS
+    results: dict[int, bool] = {}
+    with ThreadPoolExecutor(max_workers=workers) as pool:
+        futures = {pool.submit(_is_open, host, port, timeout): port for port in ports}
+        for future in as_completed(futures):
+            results[futures[future]] = future.result()
+    return results
+
+
+def _is_open(host: str, port: int, timeout: float) -> bool:
+    try:
+        with socket.create_connection((host, port), timeout=timeout):
+            return True
+    except OSError:
+        return False

backhoe/backends/shodan.py

@@ -0,0 +1,150 @@
+"""
+Shodan backend — richer per-port service/banner data than BACKHOE's own
+bounded TCP connect scan (backends/portscan.py), without doing any more
+raw TCP ourselves. Plain requests calls against Shodan's REST API, no
+`shodan` SDK dependency — same pattern as crtsh.py/gravatar.py.
+
+Needs an API key: backhoe.keys.get_api_key(SHODAN_PROVIDER) resolves one
+(SHODAN_API_KEY env var, then a locally stored key, then an interactive
+prompt — see keys.py). Shodan's free tier includes host lookups; which
+optional fields (product, version, vulns) actually come back is
+plan-dependent and NOT verified live in this environment — no Shodan key
+or live network access was available during development (see the design
+spec's "Verification honesty note"). Every optional field is read with
+.get(), never assumed present. The `vulns` field's shape in particular
+(list of CVE strings vs. a dict keyed by CVE ID) isn't confirmed from
+Shodan's own docs either — _extract_vuln_ids() below handles both rather
+than guessing one.
+
+validate_key() hits /api-info, confirmed via Shodan's own docs to cost no
+query credit — this is what lets keys.py live-validate a stored key on
+every single call without burning the operator's quota.
+"""
+from __future__ import annotations
+
+import requests
+
+from ..keys import KeyProvider, KeyValidationError
+from ..schema import Finding, FindingType
+
+BASE_URL = "https://api.shodan.io"
+DEFAULT_TIMEOUT = 10
+
+
+class ShodanError(Exception):
+    """A real failure calling Shodan — never raised for "no data for this
+    host", which is a legitimate empty result (see lookup_host)."""
+
+
+class ShodanAPIError(ShodanError):
+    """lookup_host() failed for a reason other than "not indexed": bad key,
+    rate limited, Shodan's own server error, network failure, or an
+    unparseable response."""
+
+
+class ShodanValidationError(ShodanError, KeyValidationError):
+    """validate_key() itself failed inconclusively (network error, 5xx,
+    unparseable response) — NOT the same as the key being confirmed wrong.
+    keys.py treats this as "can't confirm, use the key anyway", not as a
+    reason to reprompt."""
+
+
+def _redact_key(text: str, key: str) -> str:
+    """Never let the raw API key reach a message a caller might print
+    (click.secho, terminal scrollback, logs) — `requests` embeds the
+    full request URL, key querystring included, in a ConnectionError's
+    own message."""
+    return text.replace(key, "<redacted>") if key else text
+
+
+def validate_key(key: str) -> bool:
+    try:
+        resp = requests.get(f"{BASE_URL}/api-info", params={"key": key}, timeout=DEFAULT_TIMEOUT)
+    except requests.RequestException as exc:
+        raise ShodanValidationError(f"network error contacting Shodan: {_redact_key(str(exc), key)}") from exc
+
+    if resp.status_code == 200:
+        return True
+    if resp.status_code == 401:
+        return False
+    raise ShodanValidationError(f"unexpected Shodan /api-info status {resp.status_code}")
+
+
+def lookup_host(ip: str, key: str) -> list[Finding]:
+    try:
+        resp = requests.get(f"{BASE_URL}/shodan/host/{ip}", params={"key": key}, timeout=DEFAULT_TIMEOUT)
+    except requests.RequestException as exc:
+        raise ShodanAPIError(f"network error contacting Shodan: {_redact_key(str(exc), key)}") from exc
+
+    if resp.status_code == 404:
+        return []
+    if resp.status_code != 200:
+        raise ShodanAPIError(f"Shodan host lookup returned status {resp.status_code}")
+
+    try:
+        data = resp.json()
+    except ValueError as exc:
+        raise ShodanAPIError(f"unparseable JSON from Shodan: {exc}") from exc
+
+    return _to_findings(ip, data)
+
+
+def _extract_vuln_ids(entry: dict) -> list[str]:
+    """CVE ids for one data[] entry. Shape not verified live (see module
+    docstring) — handle both a dict keyed by CVE id and a plain list of
+    CVE id strings rather than assuming one."""
+    vulns = entry.get("vulns")
+    if not vulns:
+        return []
+    if isinstance(vulns, dict):
+        return sorted(vulns.keys())
+    if isinstance(vulns, list):
+        return sorted(str(v) for v in vulns)
+    return []
+
+
+def _to_findings(ip: str, data: dict) -> list[Finding]:
+    # Shodan can return multiple data[] entries for the same port (different
+    # banners/modules/vhosts observed on it). Group by port BEFORE building
+    # Finding objects so each port produces exactly one Finding — otherwise
+    # two same-port entries would share the same dedup key AND the same
+    # source string ("shodan"), and merge_findings() would let the second
+    # entry's raw payload silently overwrite the first's (e.g. losing a CVE).
+    by_port: dict[int, dict] = {}
+    for entry in data.get("data", []):
+        port = entry.get("port")
+        if port is None:
+            continue
+        merged = by_port.setdefault(
+            port, {"service": None, "product": None, "version": None, "vulns": []}
+        )
+        service = (entry.get("_shodan") or {}).get("module")
+        if merged["service"] is None and service is not None:
+            merged["service"] = service
+        if merged["product"] is None and entry.get("product") is not None:
+            merged["product"] = entry.get("product")
+        if merged["version"] is None and entry.get("version") is not None:
+            merged["version"] = entry.get("version")
+        for v in _extract_vuln_ids(entry):
+            if v not in merged["vulns"]:
+                merged["vulns"].append(v)
+
+    findings: list[Finding] = []
+    for port, payload in by_port.items():
+        findings.append(
+            Finding(
+                type=FindingType.OPEN_PORT,
+                value=f"{ip}:{port}",
+                source="shodan",
+                raw={"port": port, **payload},
+            )
+        )
+    return findings
+
+
+SHODAN_PROVIDER = KeyProvider(
+    name="shodan",
+    env_var="SHODAN_API_KEY",
+    prompt_label="Shodan API key",
+    validate=validate_key,
+)