rimscan-cli 1.0.0__py3-none-any.whl

rimscan_cli/recon/resolver.py

@@ -0,0 +1,232 @@
+"""DNS resolution with CNAME chain tracking and wildcard detection."""
+
+from __future__ import annotations
+
+import copy
+import random
+import string
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Dict, List, Optional, Sequence, Set, Tuple
+
+import dns.exception
+import dns.resolver
+
+from rimscan_cli.recon._common import normalize_name
+
+__all__ = ["ResolutionStatus", "ResolutionResult", "DNSResolver"]
+
+
+class ResolutionStatus(Enum):
+    A_RECORD = "a_record"
+    CNAME = "cname"
+    DANGLING_CNAME = "dangling_cname"
+    NXDOMAIN = "nxdomain"
+    NO_RECORD = "no_record"
+    WILDCARD = "wildcard"
+    TIMEOUT = "timeout"
+    ERROR = "error"
+
+
+# Transient failures are not cached so a flaky lookup can be retried later.
+_CACHEABLE = {
+    ResolutionStatus.A_RECORD,
+    ResolutionStatus.CNAME,
+    ResolutionStatus.DANGLING_CNAME,
+    ResolutionStatus.NXDOMAIN,
+    ResolutionStatus.NO_RECORD,
+    ResolutionStatus.WILDCARD,
+}
+
+
+@dataclass
+class ResolutionResult:
+    hostname: str
+    status: ResolutionStatus
+    a_records: List[str] = field(default_factory=list)
+    cname_target: Optional[str] = None
+    cname_chain: List[str] = field(default_factory=list)
+    error_detail: Optional[str] = None
+
+    def __repr__(self) -> str:
+        if self.status in (ResolutionStatus.CNAME, ResolutionStatus.DANGLING_CNAME):
+            return f"<{self.hostname} {self.status.value} via {' -> '.join(self.cname_chain)}>"
+        if self.status is ResolutionStatus.A_RECORD:
+            return f"<{self.hostname} A {', '.join(self.a_records)}>"
+        return f"<{self.hostname} {self.status.value}>"
+
+    @property
+    def is_live(self) -> bool:
+        """True when the name resolves to at least one address and is not wildcard noise."""
+        return bool(self.a_records) and self.status in (
+            ResolutionStatus.A_RECORD,
+            ResolutionStatus.CNAME,
+        )
+
+
+class DNSResolver:
+    def __init__(
+        self,
+        timeout: float = 5.0,
+        max_workers: int = 20,
+        nameservers: Optional[Sequence[str]] = None,
+    ):
+        self.max_workers = max_workers
+        self._resolver = dns.resolver.Resolver()
+        self._resolver.timeout = timeout
+        self._resolver.lifetime = timeout
+        if nameservers:
+            self._resolver.nameservers = list(nameservers)
+        self._cache: Dict[str, ResolutionResult] = {}
+        self._wildcard_domain: Optional[str] = None
+        self._wildcard_ips: Set[str] = set()
+        self._wildcard_cnames: Set[str] = set()
+
+    # -- wildcard handling -------------------------------------------------
+
+    def detect_wildcard(self, domain: str, probes: int = 3, min_hits: int = 2) -> bool:
+        """Resolve random labels under `domain`; if several resolve, it is a wildcard zone.
+
+        CDN-backed wildcards can answer with different addresses for every label, so
+        the probes only have to resolve, not agree. Call this before resolving hosts.
+
+        Limitation: a real host that shares its addresses or CNAME target with the
+        wildcard cannot be told apart from wildcard noise and is reported as WILDCARD.
+        """
+        domain = normalize_name(domain)
+        self._wildcard_domain = None
+        self._wildcard_ips = set()
+        self._wildcard_cnames = set()
+        self._cache.clear()
+
+        hits: List[ResolutionResult] = []
+        for _ in range(probes):
+            label = "".join(random.choices(string.ascii_lowercase + string.digits, k=16))
+            result = self._resolve_uncached(f"{label}.{domain}")
+            if result.is_live:
+                hits.append(result)
+
+        if len(hits) < min(min_hits, probes):
+            return False
+        self._wildcard_domain = domain
+        for hit in hits:
+            self._wildcard_ips.update(hit.a_records)
+            if hit.cname_chain:
+                self._wildcard_cnames.add(hit.cname_chain[-1])
+        return True
+
+    def _is_wildcard_answer(self, hostname: str, ips: List[str], chain: List[str]) -> bool:
+        domain = self._wildcard_domain
+        if not domain or not hostname.endswith("." + domain):
+            return False
+        if chain and chain[-1] in self._wildcard_cnames:
+            return True
+        return bool(ips) and set(ips) <= self._wildcard_ips
+
+    # -- lookups -----------------------------------------------------------
+
+    def _lookup(self, name: str, rdtype: str) -> Tuple[list, str]:
+        """Return (records, state); state is ok, nodata, nxdomain, timeout or error."""
+        for attempt in range(2):
+            try:
+                return list(self._resolver.resolve(name, rdtype)), "ok"
+            except dns.resolver.NoAnswer:
+                return [], "nodata"
+            except dns.resolver.NXDOMAIN:
+                return [], "nxdomain"
+            except dns.exception.Timeout:
+                if attempt == 1:
+                    return [], "timeout"
+            except dns.exception.DNSException:
+                return [], "error"
+        return [], "timeout"
+
+    @staticmethod
+    def _failure(hostname: str, chain: List[str], state: str) -> ResolutionResult:
+        if state == "nxdomain":
+            if chain:
+                return ResolutionResult(
+                    hostname, ResolutionStatus.DANGLING_CNAME, cname_chain=chain, cname_target=chain[-1]
+                )
+            return ResolutionResult(hostname, ResolutionStatus.NXDOMAIN)
+        if state == "timeout":
+            return ResolutionResult(hostname, ResolutionStatus.TIMEOUT, cname_chain=chain)
+        return ResolutionResult(
+            hostname, ResolutionStatus.ERROR, cname_chain=chain, error_detail="DNS lookup failed"
+        )
+
+    def _resolve_uncached(self, hostname: str, max_chain_depth: int = 10) -> ResolutionResult:
+        chain: List[str] = []
+        current = normalize_name(hostname)
+        visited = {current}
+
+        while True:
+            records, state = self._lookup(current, "CNAME")
+            if state == "nodata":
+                break
+            if state != "ok":
+                return self._failure(hostname, chain, state)
+            target = normalize_name(str(records[0].target))
+            if target in visited:
+                return ResolutionResult(
+                    hostname, ResolutionStatus.ERROR, cname_chain=chain,
+                    error_detail=f"CNAME loop at {target}",
+                )
+            if len(chain) >= max_chain_depth:
+                return ResolutionResult(
+                    hostname, ResolutionStatus.ERROR, cname_chain=chain,
+                    error_detail=f"CNAME chain longer than {max_chain_depth} hops",
+                )
+            visited.add(target)
+            chain.append(target)
+            current = target
+
+        ips: List[str] = []
+        for rdtype in ("A", "AAAA"):
+            records, state = self._lookup(current, rdtype)
+            if state == "ok":
+                ips = [str(r) for r in records]
+                break
+            if state != "nodata":
+                return self._failure(hostname, chain, state)
+
+        if not ips:
+            if chain:
+                return ResolutionResult(
+                    hostname, ResolutionStatus.CNAME, cname_chain=chain, cname_target=chain[-1]
+                )
+            return ResolutionResult(hostname, ResolutionStatus.NO_RECORD)
+
+        if self._is_wildcard_answer(normalize_name(hostname), ips, chain):
+            return ResolutionResult(
+                hostname, ResolutionStatus.WILDCARD, a_records=ips, cname_chain=chain
+            )
+        if chain:
+            return ResolutionResult(
+                hostname, ResolutionStatus.CNAME, a_records=ips,
+                cname_chain=chain, cname_target=chain[-1],
+            )
+        return ResolutionResult(hostname, ResolutionStatus.A_RECORD, a_records=ips)
+
+    # -- public API --------------------------------------------------------
+
+    def resolve_one(self, hostname: str, max_chain_depth: int = 10) -> ResolutionResult:
+        key = normalize_name(hostname)
+        cached = self._cache.get(key)
+        if cached is not None:
+            result = copy.deepcopy(cached)
+            result.hostname = hostname
+            return result
+        result = self._resolve_uncached(hostname, max_chain_depth)
+        if result.status in _CACHEABLE:
+            self._cache[key] = copy.deepcopy(result)
+        return result
+
+    def resolve_many(self, hostnames: Sequence[str]) -> List[ResolutionResult]:
+        """Resolve hosts concurrently; results follow the order of `hostnames`."""
+        names = list(dict.fromkeys(hostnames))
+        if not names:
+            return []
+        with ThreadPoolExecutor(max_workers=min(self.max_workers, len(names))) as pool:
+            return list(pool.map(self.resolve_one, names))

rimscan_cli/recon/secret_scan.py

@@ -0,0 +1,241 @@
+"""Detect credentials exposed in publicly served files."""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import math
+import re
+from collections import Counter
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, field
+from typing import Dict, Iterable, List, Optional, Pattern, Set, Tuple
+
+import requests
+
+from rimscan_cli.recon._http import make_session, read_capped
+
+__all__ = ["SecretFinding", "SecretScanner", "SECRET_PATTERNS", "COMMON_PATHS", "MAX_HOSTS"]
+
+logger = logging.getLogger(__name__)
+
+COMMON_PATHS = (
+    "/.env",
+    "/.env.local",
+    "/.env.production",
+    "/config.js",
+    "/app.js",
+    "/main.js",
+    "/static/js/main.js",
+    "/assets/env.js",
+    "/api/config",
+)
+
+# Config-style files are never legitimately served as HTML; an HTML answer is a soft 404.
+_NON_HTML_PATHS = {"/.env", "/.env.local", "/.env.production"}
+MAX_HOSTS = 50
+_MAX_BODY_BYTES = 1024 * 1024
+_CONTEXT_RADIUS = 40
+_AWS_CONTEXT_RADIUS = 80
+
+_PRIVATE_KEY_HEADER = re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----")
+
+SECRET_PATTERNS: Dict[str, Pattern[str]] = {
+    "aws_access_key": re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b"),
+    "aws_secret_key": re.compile(r"(?<![A-Za-z0-9/+])[A-Za-z0-9/+]{40}(?![A-Za-z0-9/+=])"),
+    "github_token": re.compile(r"\bgh[pousr]_[A-Za-z0-9]{36,}\b"),
+    "slack_token": re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{10,}"),
+    "stripe_live_key": re.compile(r"\b[sr]k_live_[0-9a-zA-Z]{24,}"),
+    "jwt": re.compile(r"\beyJ[A-Za-z0-9_-]{5,}\.eyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]*"),
+    "private_key": _PRIVATE_KEY_HEADER,
+}
+
+# A bare 40-character base64 string is too common (hashes, build ids) to report alone.
+_AWS_SECRET_CONTEXT = re.compile(r"aws|secret", re.IGNORECASE)
+
+Span = Tuple[int, int]
+
+
+def _redact(value: str) -> str:
+    """Mask a credential, keeping only its first and last four characters.
+
+    A private-key header is a public marker, not a secret, and it tells a reader what kind
+    of key was exposed (RSA, EC, OpenSSH, encrypted, ...), so it is kept whole. Anything
+    that follows it inside the same span is key material and is replaced by "[...]".
+    """
+    header = _PRIVATE_KEY_HEADER.match(value)
+    if header:
+        return header.group(0) + ("[...]" if len(value) > header.end() else "")
+    return "***" if len(value) <= 8 else f"{value[:4]}...{value[-4:]}"
+
+
+def _merge_spans(spans: Iterable[Span]) -> List[Span]:
+    merged: List[Span] = []
+    for start, end in sorted(spans):
+        if merged and start < merged[-1][1]:
+            merged[-1] = (merged[-1][0], max(merged[-1][1], end))
+        else:
+            merged.append((start, end))
+    return merged
+
+
+def _redacted_window(line: str, start: int, end: int, spans: List[Span]) -> str:
+    """Text around a match with every credential on the line redacted.
+
+    The window never cuts through a credential: if an edge lands inside one, the window
+    grows to include it whole, so no fragment of a secret is left unredacted.
+    """
+    low = max(0, start - _CONTEXT_RADIUS)
+    high = min(len(line), end + _CONTEXT_RADIUS)
+    for span_start, span_end in spans:
+        if span_start < low < span_end:
+            low = span_start
+        if span_start < high < span_end:
+            high = span_end
+    parts: List[str] = []
+    position = low
+    for span_start, span_end in spans:
+        if span_end <= low or span_start >= high:
+            continue
+        parts.append(line[position:span_start])
+        parts.append(_redact(line[span_start:span_end]))
+        position = span_end
+    parts.append(line[position:high])
+    return "".join(parts).strip()
+
+
+@dataclass
+class SecretFinding:
+    secret_type: str
+    source: str
+    line_number: Optional[int] = None
+    entropy: float = 0.0
+    # The raw value is kept out of repr() and to_dict() on purpose; `context` is built
+    # already redacted by the scanner and is redacted again on output as a second layer.
+    match: str = field(default="", repr=False)
+    context: str = field(default="", repr=False)
+
+    def redacted_match(self) -> str:
+        return _redact(self.match)
+
+    def redacted_context(self) -> str:
+        text = self.context
+        for pattern in SECRET_PATTERNS.values():
+            text = pattern.sub(lambda m: _redact(m.group()), text)
+        return text
+
+    def to_dict(self) -> dict:
+        return {
+            "secret_type": self.secret_type,
+            "match": self.redacted_match(),
+            "source": self.source,
+            "line_number": self.line_number,
+            "entropy": self.entropy,
+            "context": self.redacted_context(),
+        }
+
+
+def shannon_entropy(data: str) -> float:
+    if not data:
+        return 0.0
+    total = len(data)
+    return -sum((n / total) * math.log2(n / total) for n in Counter(data).values())
+
+
+class SecretScanner:
+    def __init__(
+        self,
+        min_entropy: float = 4.5,
+        timeout: Tuple[float, float] = (5.0, 10.0),
+        max_workers: int = 8,
+    ):
+        self.min_entropy = min_entropy
+        self.timeout = timeout
+        self.max_workers = max_workers
+        self._session = make_session(retries=1)
+
+    def scan_text(self, text: str, source: str = "unknown") -> List[SecretFinding]:
+        findings: List[SecretFinding] = []
+        for line_no, line in enumerate(text.splitlines(), start=1):
+            hits: List[Tuple[int, int, str, str, float]] = []
+            for secret_type, pattern in SECRET_PATTERNS.items():
+                for match in pattern.finditer(line):
+                    value = match.group(0)
+                    entropy = shannon_entropy(value)
+                    if secret_type == "aws_secret_key":
+                        nearby = line[max(0, match.start() - _AWS_CONTEXT_RADIUS):match.end() + _AWS_CONTEXT_RADIUS]
+                        if entropy < self.min_entropy or not _AWS_SECRET_CONTEXT.search(nearby):
+                            continue
+                    # A key embedded in JSON or JS keeps its body on the same line as the
+                    # header. Everything after the header on that line counts as part of the
+                    # secret, so the context shown for the finding never contains key material.
+                    end = len(line) if secret_type == "private_key" else match.end()
+                    hits.append((match.start(), end, secret_type, value, entropy))
+            if not hits:
+                continue
+            spans = _merge_spans((start, end) for start, end, *_ in hits)
+            for start, end, secret_type, value, entropy in hits:
+                findings.append(
+                    SecretFinding(
+                        secret_type, source, line_no, round(entropy, 2),
+                        match=value, context=_redacted_window(line, start, end, spans),
+                    )
+                )
+        return findings
+
+    def scan_domain(self, hosts: Iterable[str], max_hosts: int = MAX_HOSTS) -> List[SecretFinding]:
+        """Fetch a fixed list of well-known paths on each host and scan the responses.
+
+        Sends one GET per path in COMMON_PATHS per host, plus one more over HTTP when the
+        HTTPS request fails to connect. The session may also retry once on a 429 or 5xx
+        answer (it never waits for a server-supplied Retry-After). Redirects are not
+        followed. A request that fails, including one that breaks while its body is being
+        read, only skips that path. At most `max_hosts` hosts are scanned; a warning is
+        logged when the list is longer. A response body is reported once per host, so
+        soft-404 pages do not repeat, but the same leak on two hosts is reported for both.
+        Callers must have permission to test every host passed in.
+        """
+        host_list = list(dict.fromkeys(hosts))
+        if len(host_list) > max_hosts:
+            logger.warning("Secret scan limited to %d of %d hosts", max_hosts, len(host_list))
+            host_list = host_list[:max_hosts]
+        urls = [(host, path) for host in host_list for path in COMMON_PATHS]
+        seen_bodies: Set[Tuple[str, str]] = set()
+        findings: List[SecretFinding] = []
+
+        with ThreadPoolExecutor(max_workers=self.max_workers) as pool:
+            for host, url, body in pool.map(lambda hp: (hp[0], *self._fetch(*hp)), urls):
+                if body is None:
+                    continue
+                digest = hashlib.sha256(body.encode("utf-8", "replace")).hexdigest()
+                if (host, digest) in seen_bodies:
+                    continue
+                seen_bodies.add((host, digest))
+                findings.extend(self.scan_text(body, source=url))
+        return findings
+
+    def _fetch(self, host: str, path: str) -> Tuple[str, Optional[str]]:
+        for scheme in ("https", "http"):
+            url = f"{scheme}://{host}{path}"
+            try:
+                response = self._session.get(
+                    url, timeout=self.timeout, stream=True, allow_redirects=False
+                )
+            except requests.RequestException:
+                continue
+            if response.status_code >= 300:
+                # Errors and redirects (redirects are not followed) carry no file content.
+                response.close()
+                return url, None
+            if path in _NON_HTML_PATHS and "html" in response.headers.get("Content-Type", "").lower():
+                response.close()
+                return url, None
+            try:
+                return url, read_capped(response, _MAX_BODY_BYTES)
+            except requests.RequestException:
+                # The body is streamed, so a reset or read timeout can surface here. It must
+                # not escape the worker (pool.map would re-raise it and the scan would lose
+                # every result), and the host did answer, so there is no HTTP fallback.
+                response.close()
+                return url, None
+        return f"https://{host}{path}", None

rimscan_cli/recon/severity.py

@@ -0,0 +1,15 @@
+"""Severity levels shared by every audit module."""
+
+from __future__ import annotations
+
+from enum import Enum
+
+__all__ = ["Severity"]
+
+
+class Severity(Enum):
+    CRITICAL = "critical"
+    HIGH = "high"
+    MEDIUM = "medium"
+    LOW = "low"
+    INFO = "info"