loghunter-cli 0.1.0.dev0__py3-none-any.whl

loghunter/detectors/ssl.py

@@ -0,0 +1,25 @@
+"""SSL detector — TLS anomaly detection from Zeek ssl.log. (planned)
+
+Flags self-signed certificates, weak cipher suites, unusual SNI patterns,
+and certificate validity anomalies that may indicate malicious infrastructure.
+"""
+
+from __future__ import annotations
+
+from loghunter.common.finding import DetectorContext, Finding
+
+DETECTOR_NAME = "ssl"
+STATUS = "planned"
+
+REQUIRED_LOGS = [
+    {"source": "zeek_dir", "pattern": "ssl*.log*"},
+]
+
+OPTIONAL_LOGS: list[dict] = []
+
+DEFAULT_CONFIG: dict = {}
+
+
+def run(context: DetectorContext) -> list[Finding]:
+    """Detect TLS anomalies including self-signed certs and cipher outliers."""
+    raise NotImplementedError("ssl detector is planned — not yet implemented")

loghunter/detectors/syslog.py

@@ -0,0 +1,266 @@
+"""Syslog anomaly detector — drain3 templating + rarity scoring.
+
+Pipeline:
+1. drain3 log templating: assigns each message row a template_id and template_str
+2. Rarity scoring: flags templates whose occurrence count falls at or below
+   min(percentile_threshold, max_count) as anomalous
+3. Reboot detection: scans all rows for known reboot/shutdown signal patterns
+4. Reboot suppression: anomalous events within reboot_suppress_window seconds after
+   a detected reboot on the same host are suppressed; one synthetic reboot annotation
+   is emitted per detected reboot in their place
+5. Finding production: one Finding per anomalous event plus one per synthetic reboot
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from datetime import datetime, timezone
+from typing import Any
+
+import pandas as pd
+from tqdm import tqdm
+
+from loghunter.common.finding import DetectorContext, Finding, MethodTag, Severity
+
+DETECTOR_NAME = "syslog"
+STATUS = "available"
+
+# syslog is fidelity-aware: either flat rsyslog (syslog_dir/*.log*) OR Zeek's
+# own syslog.log (zeek_dir/syslog*.log*). At least one must be present; both
+# concat before drain3. Detector is SOURCE-BLIND — references only the
+# minimal-5 (ts, host, program, raw, message). Zeek's extended facility /
+# severity ride along on the frame but are NEVER read here; the digest
+# consumes them. Mirrors the dns detector's Zeek + Pi-hole shape.
+REQUIRED_LOGS: list[dict] = []
+
+OPTIONAL_LOGS = [
+    {"source": "syslog_dir", "pattern": "*.log*"},
+    {"source": "zeek_dir",   "pattern": "syslog*.log*"},
+]
+
+REQUIRES_ONE_OF_OPTIONAL = True
+REQUIRES_ONE_OF_OPTIONAL_REASON = (
+    "syslog — no syslog source found "
+    "(need syslog_dir files or zeek_dir syslog.log)"
+)
+
+DRAIN_SIM_THRESH          = 0.5
+DRAIN_DEPTH               = 4
+DRAIN_PARAMETRIZE_NUMERIC = True
+REBOOT_SUPPRESS_WINDOW    = 300  # seconds
+
+DEFAULT_CONFIG = {
+    "lookback_days":          7,
+    "rarity_pct":            10,
+    "max_count":              1,
+    "sim_thresh":             DRAIN_SIM_THRESH,
+    "depth":                  DRAIN_DEPTH,
+    "parametrize_numeric":    DRAIN_PARAMETRIZE_NUMERIC,
+    "reboot_suppress_window": REBOOT_SUPPRESS_WINDOW,
+}
+
+DETECTOR_METHOD = MethodTag("drain3", named=True)
+
+
+def run(context: DetectorContext) -> list[Finding]:
+    """Detect anomalous syslog lines using drain3 templating and rarity scoring."""
+    flat_df = context.logs.get("*.log*")
+    zeek_df = context.logs.get("syslog*.log*")
+
+    frames = [df for df in (flat_df, zeek_df) if df is not None and not df.empty]
+    if not frames:
+        return []
+    df = frames[0] if len(frames) == 1 else pd.concat(frames, ignore_index=True)
+
+    cfg = context.config
+    sim_thresh      = cfg.get("sim_thresh",             DEFAULT_CONFIG["sim_thresh"])
+    depth           = cfg.get("depth",                  DEFAULT_CONFIG["depth"])
+    parametrize     = cfg.get("parametrize_numeric",    DEFAULT_CONFIG["parametrize_numeric"])
+    rarity_pct      = cfg.get("rarity_pct",             DEFAULT_CONFIG["rarity_pct"])
+    max_count       = cfg.get("max_count",              DEFAULT_CONFIG["max_count"])
+    suppress_window = cfg.get("reboot_suppress_window", DEFAULT_CONFIG["reboot_suppress_window"])
+
+    df = _run_drain3(df, sim_thresh, depth, parametrize)
+    df, threshold, freq = _score_rarity(df, rarity_pct, max_count)
+    reboots = _detect_reboots(df)
+    anomaly_df = df[df["is_anomaly"]].copy()
+    kept_df, synthetic_records = _apply_suppression(anomaly_df, reboots, suppress_window)
+
+    now = datetime.now(timezone.utc)
+    timestamped: list[tuple[float, Finding]] = []
+
+    for row in kept_df.itertuples():
+        ts_sort = float("inf") if pd.isna(row.ts) else float(row.ts)
+        f = Finding(
+            detector=DETECTOR_NAME,
+            severity=Severity.MEDIUM,
+            title=str(row.raw)[:180],
+            description="Rare log template observed at or below rarity threshold",
+            evidence={
+                "host":         row.host,
+                "template_id":  int(row.template_id),
+                "template_str": row.template_str,
+                "count":        int(freq[int(row.template_id)]),
+                "threshold":    int(threshold),
+            },
+            next_steps=[
+                "Review surrounding log context for this host",
+                "Check if template appears in recent incidents",
+            ],
+            ts_generated=now,
+            data_window=context.data_window,
+        )
+        timestamped.append((ts_sort, f))
+
+    for record in synthetic_records:
+        reboot_ts = record["ts"]
+        ts_sort = reboot_ts.timestamp() if reboot_ts is not None else float("inf")
+        f = Finding(
+            detector=DETECTOR_NAME,
+            severity=Severity.INFO,
+            title=record["raw"],
+            description="Reboot detected — anomalous events within suppression window are excluded",
+            evidence={
+                "host":                      record["host"],
+                "reboot_ts":                 reboot_ts.isoformat() if reboot_ts is not None else None,
+                "suppressed_window_seconds": suppress_window,
+            },
+            next_steps=["Review system logs around the reboot time for pre-reboot anomalies"],
+            ts_generated=now,
+            data_window=context.data_window,
+        )
+        timestamped.append((ts_sort, f))
+
+    timestamped.sort(key=lambda x: x[0])
+    return [f for _, f in timestamped]
+
+
+def _run_drain3(
+    df: pd.DataFrame,
+    sim_thresh: float,
+    depth: int,
+    parametrize_numeric: bool,
+) -> pd.DataFrame:
+    """Add template_id and template_str columns via drain3 log templating."""
+    try:
+        from drain3 import TemplateMiner
+        from drain3.template_miner_config import TemplateMinerConfig
+    except ImportError:
+        raise ImportError(
+            "drain3 is required for syslog detection. Run: pip install drain3"
+        )
+
+    cfg = TemplateMinerConfig()
+    cfg.drain_sim_th               = sim_thresh
+    cfg.drain_depth                = depth
+    cfg.parametrize_numeric_tokens = parametrize_numeric
+
+    miner = TemplateMiner(config=cfg)
+    template_ids: list[int] = []
+    template_strs: list[str] = []
+
+    # leave=True + clean bar_format makes this the liveness narration for the
+    # syslog detector phase (the runner deliberately skips its outer spinner
+    # for syslog so the two writers don't fight for the same stderr line).
+    for msg in tqdm(
+        df["message"],
+        desc="syslog: mining templates",
+        unit=" lines",
+        unit_scale=True,
+        leave=True,
+        bar_format="{desc}: {n_fmt} lines [{elapsed}]",
+    ):
+        result = miner.add_log_message(str(msg))
+        template_ids.append(result["cluster_id"])
+        template_strs.append(result["template_mined"])
+
+    df = df.copy()
+    df["template_id"]  = template_ids
+    df["template_str"] = template_strs
+    return df
+
+
+def _score_rarity(
+    df: pd.DataFrame,
+    rarity_pct: int,
+    max_count: int,
+) -> tuple[pd.DataFrame, int, dict[int, int]]:
+    """Add is_anomaly column; return (df, effective_threshold, freq_dict)."""
+    freq: dict[int, int] = {
+        int(k): int(v) for k, v in df["template_id"].value_counts().items()
+    }
+
+    sorted_counts = sorted(freq.values())
+    idx           = max(0, int(len(sorted_counts) * rarity_pct / 100) - 1)
+    pct_threshold = sorted_counts[idx]
+    threshold     = min(pct_threshold, max_count)
+
+    rare_ids = {tid for tid, count in freq.items() if count <= threshold}
+
+    df = df.copy()
+    df["is_anomaly"] = df["template_id"].map(lambda tid: int(tid) in rare_ids)
+    return df, threshold, freq
+
+
+def _detect_reboots(df: pd.DataFrame) -> dict[str, list[datetime]]:
+    """Return {host: [reboot_datetimes]} by scanning all rows for reboot signals."""
+    from loghunter.parsers.syslog import is_reboot_signal
+
+    reboots: dict[str, list[datetime]] = defaultdict(list)
+
+    for row in df.itertuples():
+        if pd.isna(row.ts):
+            continue
+        if is_reboot_signal(str(row.raw)):
+            dt = datetime.fromtimestamp(float(row.ts), tz=timezone.utc)
+            reboots[row.host].append(dt)
+
+    return {host: sorted(times) for host, times in reboots.items()}
+
+
+def _apply_suppression(
+    anomaly_df: pd.DataFrame,
+    reboots: dict[str, list[datetime]],
+    suppress_window: int,
+) -> tuple[pd.DataFrame, list[dict[str, Any]]]:
+    """Suppress anomalous events near reboots; emit one synthetic record per reboot.
+
+    Returns (kept_df, synthetic_reboot_records).
+    """
+    kept_indices: list[Any] = []
+    synthetic_records: list[dict[str, Any]] = []
+    emitted_reboots: set[tuple[str, datetime]] = set()
+
+    for row in anomaly_df.itertuples():
+        host = row.host
+        ts   = row.ts
+
+        if pd.isna(ts) or host not in reboots:
+            kept_indices.append(row.Index)
+            continue
+
+        event_dt   = datetime.fromtimestamp(float(ts), tz=timezone.utc)
+        suppressed = False
+
+        for reboot_dt in reboots[host]:
+            delta = (event_dt - reboot_dt).total_seconds()
+            if 0 <= delta <= suppress_window:
+                key = (host, reboot_dt)
+                if key not in emitted_reboots:
+                    emitted_reboots.add(key)
+                    synthetic_records.append({
+                        "ts":        reboot_dt,
+                        "host":      host,
+                        "raw":       (
+                            f"*** {host} rebooted at "
+                            f"{reboot_dt.strftime('%a %b %d %H:%M:%S')} ***"
+                        ),
+                        "synthetic": True,
+                    })
+                suppressed = True
+                break
+
+        if not suppressed:
+            kept_indices.append(row.Index)
+
+    return anomaly_df.loc[kept_indices], synthetic_records

loghunter/detectors/weird.py

@@ -0,0 +1,27 @@
+"""Weird detector — signals from Zeek weird.log and notice.log. (planned)
+
+Surfaces Zeek's own anomaly signals: protocol violations, connection state issues,
+and notice events that indicate suspicious or malformed network behavior.
+"""
+
+from __future__ import annotations
+
+from loghunter.common.finding import DetectorContext, Finding
+
+DETECTOR_NAME = "weird"
+STATUS = "planned"
+
+REQUIRED_LOGS = [
+    {"source": "zeek_dir", "pattern": "weird*.log*"},
+]
+
+OPTIONAL_LOGS = [
+    {"source": "zeek_dir", "pattern": "notice*.log*"},
+]
+
+DEFAULT_CONFIG: dict = {}
+
+
+def run(context: DetectorContext) -> list[Finding]:
+    """Aggregate and score Zeek weird and notice events."""
+    raise NotImplementedError("weird detector is planned — not yet implemented")

loghunter/digest/__init__.py

@@ -0,0 +1,43 @@
+"""digest verb — orient-before-the-hunt.
+
+A digest characterises the dominant shape of a log pile and states facts
+about it. It is a peer verb to detect, not a detector. Digest never produces
+a Finding and never reaches a verdict.
+
+One summariser module per schema. Each summariser is a function
+``summarize(frame) -> dict`` returning the schema-specific body of a
+DigestCard. The dispatcher below imports the right module by schema name; to
+add a new schema, drop a new module beside conn.py and nothing else changes.
+
+Architectural rail: digest consumes the loaded frame BEFORE the allowlist
+filtering seam. Allowlisted infrastructure (resolvers, pollers) is part of
+what's in the pile and stays on the sonar. The digest call graph must not
+touch build_matcher or AllowlistMatcher.filter_df.
+"""
+
+from __future__ import annotations
+
+from importlib import import_module
+from typing import Callable
+
+import pandas as pd
+
+
+def get_summarizer(schema: str) -> Callable[..., dict]:
+    """Return the summarize() function for a given digest schema.
+
+    The dispatcher returns the bare callable; per-schema signatures may
+    differ. Today: ``conn`` and ``cloudtrail`` take ``(frame)``;
+    ``dns`` takes ``(frame, feed)`` where feed is ``"zeek"`` or
+    ``"pihole"``; ``syslog`` takes ``(frame, feed)`` where feed is
+    ``"zeek"`` or ``"syslog"`` (flat rsyslog). Callers (currently only
+    ``run_digest``) know how to invoke the right signature per schema.
+
+    Raises ValueError with an actionable message when no summariser
+    exists for the requested schema.
+    """
+    try:
+        module = import_module(f"loghunter.digest.{schema}")
+    except ModuleNotFoundError as exc:
+        raise ValueError(f"digest: no summarizer for schema {schema!r}") from exc
+    return module.summarize

loghunter/digest/_stats.py

@@ -0,0 +1,182 @@
+"""Shared digest stats — the home for primitives that more than one card needs.
+
+Hosts:
+  - ``_rate``  / ``RATE_FLOOR``  — fraction-of-events-matching-a-kind statistic
+                                   with top-contributor attribution
+  - ``_share`` / ``SHARE_GATE``  — concentration-against-total statistic with no
+                                   population floor
+  - ``select_insights_and_fields`` — shared selection helper that promotes the
+                                     top-N speaking gated slots to insights and
+                                     returns the leftover slots as fields
+
+Cliff machinery (``_cliff``, ``CLIFF_GATE``, ``CLIFF_DISPLAY_CAP``,
+``POPULATION_FLOOR``, ``_format_ratio_cell``, ``_format_ratio_lede``) lives in
+``loghunter.digest.conn`` and stays there — that is the established shared
+origin and every card already imports cliff helpers from it.
+
+The trigger for factoring into this module is "three identical real uses"
+(``_rate``, now imported by dns + syslog + cloudtrail) or "shared by the new
+statistic by design" (``_share`` introduced by the cloudtrail source-ip slot
+and reusable by any future concentration-against-total slot). Tail
+(``_tail`` / ``TAIL_GATE``) stays local to dns.py — one-use primitives do not
+belong here yet.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable
+
+import pandas as pd
+
+from loghunter.common.finding import DigestSlot
+from loghunter.digest.conn import POPULATION_FLOOR
+
+
+# ── Calibration constants ───────────────────────────────────────────────────
+
+RATE_FLOOR = 0.01    # fraction below this → rate slots dash. Pure presence
+                     # floor, NOT a badness threshold — meaning is the same on
+                     # every network. Calibratable here.
+
+SHARE_GATE = 0.80    # top-share at or above this → share slots speak. The
+                     # share statistic exists to surface concentration; this
+                     # gate is the concentration threshold. There is no
+                     # paired population floor — see ``_share`` below.
+
+
+# ── Rate statistic ──────────────────────────────────────────────────────────
+#
+# Behavior must NOT change from the previous in-card definitions — that is the
+# proof-of-correctness for the factoring. Identical body to the three card
+# copies it replaces.
+
+def _rate(kind_mask: pd.Series, contributor_series: pd.Series) -> tuple | None:
+    """Rate statistic: what fraction of events are of a notable kind?
+
+    Returns ``(fraction, top_contributor)`` when speaking, None when dashed.
+    Dashes when total < POPULATION_FLOOR or fraction < RATE_FLOOR. The floor
+    is a pure presence bar — meaning the same on every network, never a
+    badness judgment.
+    """
+    total = len(kind_mask)
+    if total < POPULATION_FLOOR:
+        return None
+    kind_count = int(kind_mask.sum())
+    if kind_count == 0:
+        return None
+    fraction = kind_count / total
+    if fraction < RATE_FLOOR:
+        return None
+    matching = contributor_series[kind_mask].dropna()
+    if matching.empty:
+        return None
+    top = matching.value_counts().idxmax()
+    return fraction, str(top)
+
+
+# ── Share statistic ─────────────────────────────────────────────────────────
+
+def _share(sorted_counts: pd.Series, total: int) -> tuple[Any, float] | None:
+    """Share statistic: is one entity's count a dominant fraction of the total?
+
+    Returns ``(rank1_entity, top_share)`` when speaking, None when dashed.
+    Dashes only when ``top_share < SHARE_GATE`` — there is NO population
+    floor. The slot using this statistic exists to surface concentration
+    against the total, and low entity cardinality is the SIGNAL, not noise:
+    a pile of ONE distinct value at 100% MUST speak (top_share == 1.0); two
+    distinct entities with one at 99% MUST speak. Adding a population floor
+    here would suppress the exact attack shape the share slot was introduced
+    to catch.
+
+    ``sorted_counts`` must be descending; the caller's value_counts output is
+    already that shape. ``total`` is the caller-supplied denominator — for
+    source-ip in cloudtrail that is the interactive-event count, NOT a
+    derived sum, so the share is measured against the lane the caller meant.
+
+    Defensive returns:
+      - empty series or non-positive total → None
+      - NaN rank1 → None
+    """
+    if total <= 0 or len(sorted_counts) == 0:
+        return None
+    rank1 = sorted_counts.iloc[0]
+    if pd.isna(rank1):
+        return None
+    top_share = float(rank1) / float(total)
+    if top_share < SHARE_GATE:
+        return None
+    return sorted_counts.index[0], top_share
+
+
+# ── Insight selection ───────────────────────────────────────────────────────
+#
+# Shared by all four schema summarisers. Identical mechanic across cards:
+# filter speaking gated slots, sort by per-statistic salience, take top-3,
+# format via the per-card formatter map → those become insights. Everything
+# else with cells goes to fields. A promoted slot is suppressed from fields.
+
+_INSIGHT_TOP_N = 3
+_GATING_STATISTICS = frozenset({"cliff", "tail", "rate", "share"})
+
+
+def _salience(slot: DigestSlot) -> float:
+    """Per-statistic salience on one comparable scale.
+
+    cliff/tail use the rank-ratio directly. rate slots are stored with the
+    percentage as magnitude (e.g. 1.0 for 1%), so dividing by
+    ``RATE_FLOOR * 100`` puts a rate slot's salience on the same scale as a
+    cliff ratio (1% over a 1% floor scores 1.0, comparable to a 1x cliff).
+    share is stored as percentage 0–100; a heavily concentrated single
+    source IS one of the most salient things on a card, so its raw
+    percentage ranks above typical cliff ratios.
+    """
+    if slot.statistic in {"cliff", "tail"}:
+        return slot.ratio or 0.0
+    if slot.statistic == "rate":
+        return (slot.magnitude or 0.0) / (RATE_FLOOR * 100.0)
+    if slot.statistic == "share":
+        return slot.magnitude or 0.0
+    return 0.0
+
+
+def select_insights_and_fields(
+    slots: list[DigestSlot],
+    formatters: dict[str, Callable[[DigestSlot], str]],
+) -> tuple[list[str], list[DigestSlot]]:
+    """Promote speaking gated slots to insights; return leftover speaking
+    slots as fields.
+
+    Suppression rule (Glenn's precision ask): a slot is removed from
+    ``fields`` ONLY when it actually produced an insight — i.e. it was in
+    the top-N speaking gated set AND its label had a formatter that ran.
+    A gating slot whose label has no formatter falls through to fields
+    instead of vanishing, preserving "each fact appears exactly once."
+
+    Dist slots (statistic not in the gating set) never produce insights;
+    they pass straight through to fields if they have cells.
+    Non-speaking slots (cells is None) are omitted from both insights
+    and fields.
+
+    Slot labels are unique within a card, so label-based suppression is
+    safe — no ``id()`` ceremony.
+    """
+    speaking_gated = [
+        s for s in slots
+        if s.cells is not None and s.statistic in _GATING_STATISTICS
+    ]
+    speaking_gated.sort(key=_salience, reverse=True)
+
+    promoted_labels: set[str] = set()
+    insights: list[str] = []
+    for s in speaking_gated[:_INSIGHT_TOP_N]:
+        fmt = formatters.get(s.label)
+        if fmt is None:
+            continue
+        insights.append(fmt(s))
+        promoted_labels.add(s.label)
+
+    fields = [
+        s for s in slots
+        if s.cells is not None and s.label not in promoted_labels
+    ]
+    return insights, fields