loghunter-cli 0.1.0.dev0__py3-none-any.whl

loghunter/detectors/beacon.py

@@ -0,0 +1,258 @@
+"""Beacon detector — FFT-based periodic connection detection.
+
+Algorithm:
+- Bin connection timestamps into 30-second intervals (not 10s — 10s bins place a 60s
+  beacon at the Nyquist limit, producing harmonic artifacts)
+- Compute FFT over the binned time grid (resilient to data gaps vs raw inter-arrival)
+- Composite score: 40% spectral ratio + 40% peak prominence + 20% inverted jitter CV
+- Peak prominence: peak power relative to local spectral noise floor, normalized at 100x
+- Jitter CV computed on outlier-cleaned inter-arrival deltas
+- Minimum 20 connections per candidate flow
+
+Reference calibration: MRTG 60s SSH poll scores ~0.608, dominant period exactly 60.0s.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+
+import numpy as np
+
+from loghunter.common.finding import DetectorContext, Finding, MethodTag, Severity
+
+DETECTOR_NAME = "beacon"
+STATUS = "available"
+
+REQUIRED_LOGS = [
+    {"source": "zeek_dir", "pattern": "conn*.log*"},
+]
+
+OPTIONAL_LOGS: list[dict] = []
+
+DEFAULT_CONFIG = {
+    "threshold": 0.5,
+    "min_connections": 20,
+    "bin_seconds": 30,
+}
+
+DETECTOR_METHOD = MethodTag("FFT", named=True)
+
+# Period range to consider (seconds). Outside this, FFT peaks are ignored.
+_MIN_PERIOD = 45
+_MAX_PERIOD = 7200
+
+
+def run(context: DetectorContext) -> list[Finding]:
+    """Detect beaconing flows using FFT on binned connection timestamps."""
+    cfg = context.config
+    threshold: float = cfg.get("threshold", DEFAULT_CONFIG["threshold"])
+    min_conns: int = cfg.get("min_connections", DEFAULT_CONFIG["min_connections"])
+    bin_size: int = cfg.get("bin_seconds", DEFAULT_CONFIG["bin_seconds"])
+
+    df = context.logs.get("conn*.log*")
+    if df is None or df.empty:
+        return []
+
+    df = _filter_conn(df)
+    if df.empty:
+        return []
+
+    findings: list[Finding] = []
+
+    for (src, dst, port, proto), group in df.groupby(["src", "dst", "port", "proto"]):
+        if len(group) < min_conns:
+            continue
+
+        ts_arr = group["ts"].sort_values().to_numpy(dtype=float)
+        score_data = _compute_beacon_score(ts_arr, bin_size)
+        if score_data is None or score_data["beacon_score"] < threshold:
+            continue
+
+        findings.append(_make_finding(
+            str(src), str(dst), int(port), str(proto),
+            score_data, group, context.data_window,
+        ))
+
+    findings.sort(key=lambda f: f.evidence["beacon_score"], reverse=True)
+    return findings
+
+
+def _filter_conn(df: Any) -> Any:
+    """Apply standard beacon pre-filters: established conns, no multicast, local origin."""
+    import pandas as pd
+
+    df = df[df["conn_state"].isin(["SF", "S1"])].copy()
+    df = df[~df["dst"].map(_is_multicast_or_broadcast)]
+    df = df[~df["src"].str.startswith("fe80:", na=False)]
+    df = df[~df["dst"].str.startswith("fe80:", na=False)]
+    df = df[df["local_orig"] == True]  # noqa: E712
+    df = df[df["bytes"].notna()]
+    return df
+
+
+def _is_multicast_or_broadcast(ip: str) -> bool:
+    if not isinstance(ip, str):
+        return False
+    return (
+        ip.startswith("224.") or ip.startswith("239.") or
+        ip.startswith("255.") or ip.endswith(".255") or
+        ip.startswith("ff0") or ip.startswith("ff02")
+    )
+
+
+def _compute_beacon_score(
+    ts_array: np.ndarray,
+    bin_size: int = 30,
+) -> dict[str, Any] | None:
+    """Score a single flow's connection timestamps for periodic beaconing via FFT.
+
+    Returns None if the flow cannot be scored (too few points, no variance, no
+    dominant period in the configured range).
+
+    Why binning over raw inter-arrival deltas: gaps produce delta outliers that
+    corrupt FFT results; binning represents gaps as zero-count bins, preserving
+    the periodicity signal.
+
+    Why prominence alongside spectral ratio: sparse binary signals spread energy
+    across harmonics, keeping the absolute spectral ratio low even for perfectly
+    periodic flows. Prominence measures peak power above the local noise floor,
+    robust to harmonic spreading.
+    """
+    if len(ts_array) < 10:
+        return None
+
+    t_start = ts_array.min()
+    t_end = ts_array.max()
+    n_bins = int((t_end - t_start) / bin_size) + 1
+
+    bin_idx = ((ts_array - t_start) / bin_size).astype(int)
+    counts = np.zeros(n_bins)
+    np.add.at(counts, bin_idx, 1)
+
+    std = counts.std()
+    if std == 0:
+        return None
+    counts_norm = (counts - counts.mean()) / std
+
+    fft_mag = np.abs(np.fft.rfft(counts_norm))
+    freqs = np.fft.rfftfreq(n_bins, d=bin_size)
+    fft_mag[0] = 0  # zero DC component
+
+    with np.errstate(divide="ignore"):
+        periods = np.where(freqs > 0, 1.0 / freqs, np.inf)
+
+    mask = (periods >= _MIN_PERIOD) & (periods <= _MAX_PERIOD)
+    fft_masked = np.where(mask, fft_mag, 0)
+    if fft_masked.max() == 0:
+        return None
+
+    peak_idx = int(fft_masked.argmax())
+    peak_period = float(periods[peak_idx])
+    peak_power = float(fft_mag[peak_idx])
+    total_power = float(fft_mag[1:].sum())
+    if total_power == 0:
+        return None
+
+    spectral_ratio = peak_power / total_power
+
+    window = max(10, int(peak_idx * 0.05))
+    lo = max(1, peak_idx - window)
+    hi = min(len(fft_mag) - 1, peak_idx + window)
+    local = np.concatenate([fft_mag[lo:peak_idx], fft_mag[peak_idx + 1:hi + 1]])
+    noise_floor = float(np.median(local)) if len(local) > 0 else 1.0
+    prominence = peak_power / (noise_floor + 1e-10)
+    prominence_norm = min(prominence / 100.0, 1.0)
+
+    deltas = np.diff(ts_array)
+    d_mean = deltas.mean()
+    d_std = deltas.std()
+    clean_deltas = deltas[np.abs(deltas - d_mean) < 3 * d_std]
+    if len(clean_deltas) > 1 and clean_deltas.mean() > 0:
+        jitter_cv = float(clean_deltas.std() / clean_deltas.mean())
+    else:
+        jitter_cv = 1.0
+
+    beacon_score = (
+        0.4 * spectral_ratio +
+        0.4 * prominence_norm +
+        0.2 * (1.0 - min(jitter_cv, 1.0))
+    )
+
+    return {
+        "beacon_score": round(beacon_score, 4),
+        "dominant_period": round(peak_period, 1),
+        "dominant_period_m": round(peak_period / 60, 2),
+        "spectral_ratio": round(spectral_ratio, 4),
+        "prominence": round(prominence, 2),
+        "prominence_norm": round(prominence_norm, 4),
+        "jitter_cv": round(jitter_cv, 4),
+        "conn_count": len(ts_array),
+        "occupancy": round(float((counts > 0).sum()) / n_bins, 4),
+    }
+
+
+def _make_finding(
+    src: str,
+    dst: str,
+    port: int,
+    proto: str,
+    score_data: dict[str, Any],
+    group: Any,
+    data_window: tuple[datetime, datetime],
+) -> Finding:
+    score = score_data["beacon_score"]
+    period_s = score_data["dominant_period"]
+    period_m = score_data["dominant_period_m"]
+    conn_count = score_data["conn_count"]
+
+    if score >= 0.7:
+        severity = Severity.HIGH
+    elif score >= 0.5:
+        severity = Severity.MEDIUM
+    else:
+        severity = Severity.LOW
+
+    period_str = f"{period_m:.1f}m" if period_m >= 2 else f"{period_s:.0f}s"
+    title = f"{src} → {dst}:{port}/{proto}"
+
+    bytes_s = group["bytes"].dropna()
+    bytes_mean = round(float(bytes_s.mean()), 1) if len(bytes_s) > 0 else 0.0
+
+    description = (
+        f"Flow {src} → {dst}:{port}/{proto} shows periodic beaconing with a dominant "
+        f"period of {period_str} (score={score:.4f}). "
+        f"Spectral ratio: {score_data['spectral_ratio']:.4f}, "
+        f"peak prominence: {score_data['prominence']:.2f}, "
+        f"jitter CV: {score_data['jitter_cv']:.4f}. "
+        f"Mean payload: {bytes_mean:.0f} bytes."
+    )
+
+    next_steps = [
+        f"Identify the process on {src} making connections every {period_str}",
+        f"Pivot to dns.log — search for lookups resolving to {dst}",
+        f"Check {dst} on VirusTotal, Shodan, and ASN lookup",
+        f"Review full history: zeek-cut id.orig_h id.resp_h id.resp_p ts | grep '{dst}'",
+        "Use --export-allowlist to stage this flow for allowlisting if known-good",
+    ]
+
+    evidence = {
+        **score_data,
+        "period_str": period_str,
+        "src_ip": src,
+        "dst_ip": dst,
+        "dst_port": port,
+        "proto": proto,
+        "bytes_mean": bytes_mean,
+    }
+
+    return Finding(
+        detector=DETECTOR_NAME,
+        severity=severity,
+        title=title,
+        description=description,
+        evidence=evidence,
+        next_steps=next_steps,
+        ts_generated=datetime.now(timezone.utc),
+        data_window=data_window,
+    )