loghunter-cli 0.1.0.dev0__py3-none-any.whl

loghunter/exporters/splunk.py

@@ -0,0 +1,222 @@
+"""Splunk log exporter — pulls search results from Splunk REST API to local files.
+
+Invoked via: loghunter export  (or: loghunter export splunk)
+Connects to the Splunk management port (default 8089), runs hourly-chunked oneshot
+queries, and writes results as flat syslog text to the configured output file.
+
+Credentials (in priority order):
+  LOGHUNTER_SPLUNK_USER, LOGHUNTER_SPLUNK_PASS  environment variables
+  username, password in [export.splunk] config section
+
+Splunk developer/free licenses enforce a hard per-query result cap at the binary
+level that limits.conf cannot override. Hourly chunking keeps each query well
+under this ceiling. For a 7-day pull this is 168 queries.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+from tqdm import tqdm
+
+try:
+    import splunklib.client as splunk_client
+    import splunklib.results as splunk_results
+except ImportError:
+    splunk_client = None  # type: ignore[assignment]
+    splunk_results = None  # type: ignore[assignment]
+
+# RFC 3164 PRI field: <N> or <NN> or <NNN> at start of line
+PRI_RE = re.compile(r"^<\d+>")
+
+
+def is_configured(backend_cfg: dict[str, Any]) -> bool:
+    """True when [export.splunk] has a non-empty host — preserves prior auto-detect behavior."""
+    return bool(backend_cfg.get("host", "").strip())
+
+
+def summary_descriptor(backend_cfg: dict[str, Any]) -> str:
+    """Identifier shown in the final export summary's `Backend :` line."""
+    host = backend_cfg.get("host", "")
+    port = backend_cfg.get("port", "")
+    return f"{host}:{port}"
+
+
+def _get_credentials(config: dict[str, Any]) -> tuple[str, str]:
+    """Return (username, password) from env vars or config.
+
+    Environment variables take priority over config-file values.
+    """
+    user = os.environ.get("LOGHUNTER_SPLUNK_USER", "").strip() or config.get("username", "").strip()
+    passwd = os.environ.get("LOGHUNTER_SPLUNK_PASS", "").strip() or config.get("password", "").strip()
+    if not user or not passwd:
+        raise ValueError(
+            "Splunk credentials not found — set LOGHUNTER_SPLUNK_USER and "
+            "LOGHUNTER_SPLUNK_PASS, or add username/password to [export.splunk] in config"
+        )
+    return user, passwd
+
+
+def _sdk_error_message(exc: Exception, host: str, port: int) -> str:
+    """Return an actionable user-facing message for Splunk SDK failures."""
+    exc_name = exc.__class__.__name__
+    if exc_name == "AuthenticationError":
+        return (
+            "Splunk login failed — check [export.splunk].username/password in config "
+            "and LOGHUNTER_SPLUNK_USER/LOGHUNTER_SPLUNK_PASS environment overrides"
+        )
+    return (
+        f"Could not connect to Splunk management API at {host}:{port} — "
+        f"check [export.splunk].host, [export.splunk].port, network reachability, and credentials"
+    )
+
+
+def _build_hour_windows(
+    since: datetime,
+    until: datetime,
+) -> list[tuple[datetime, datetime]]:
+    """Return one-hour (start, end) pairs spanning since..until.
+
+    Both since and until are floored to their hour boundary so every emitted
+    chunk is exactly one hour — no partial-hour chunks, mirroring the migration.
+
+    Args:
+        since: Start of the window (timezone-aware or naive).
+        until: End of the window (timezone-aware or naive).
+
+    Returns:
+        List of (chunk_start, chunk_end) in the timezone of the passed-in
+        datetimes, oldest first.
+    """
+    # Use datetimes as-is — honor the tzinfo already embedded in them.
+    # Calling .astimezone() with no argument would re-express in the process
+    # timezone (UTC on a server), shifting hour boundaries away from the user's
+    # calendar day. replace() below preserves tzinfo unchanged.
+    local_since = since
+    local_until = until
+
+    # Floor both endpoints to their hour boundary
+    window_start = local_since.replace(minute=0, second=0, microsecond=0)
+    window_end   = local_until.replace(minute=0, second=0, microsecond=0)
+
+    total_hours = int((window_end - window_start).total_seconds() // 3600)
+    return [
+        (window_start + timedelta(hours=i), window_start + timedelta(hours=i + 1))
+        for i in range(max(total_hours, 0))
+    ]
+
+
+def fetch(
+    query_config: dict[str, Any],
+    splunk_config: dict[str, Any],
+    since: datetime,
+    until: datetime,
+    verbose: bool,
+    *,
+    skip_confirm: bool = False,
+) -> tuple[list[dict[str, Any]], dict[str, Any]]:
+    """Connect to Splunk and pull all rows in hourly chunks.
+
+    Args:
+        query_config: Single query stanza from config (must have "spl" key).
+        splunk_config: [export.splunk] section of config (host, port, credentials).
+        since: Start of window.
+        until: End of window.
+        verbose: Threaded from the orchestrator. The W4 grammar keeps export
+            stdout terse and level-invariant — Splunk's fetch currently
+            ignores this flag (no per-chunk chatter at level 1).
+        skip_confirm: Part of the uniform backend contract — Splunk has no
+            cost-prompt and ignores this. Accepted so the orchestrator can
+            invoke every backend with the same signature.
+
+    Returns:
+        A tuple ``(rows, fetch_meta)``:
+          - ``rows``: result rows as a flat list of dicts with at minimum _time and _raw.
+          - ``fetch_meta``: ``{"units": <hour-window count>, "unit_label": "chunks"}``
+            — used by the orchestrator to render the run-summary span string.
+    """
+    if splunk_client is None:
+        raise ValueError("splunk-sdk not installed — run: pip install loghunt[splunk]")
+
+    user, passwd = _get_credentials(splunk_config)
+    host = splunk_config.get("host", "")
+    port = int(splunk_config.get("port", 8089))
+    spl = query_config.get("spl", "")
+
+    try:
+        service = splunk_client.connect(
+            host=host,
+            port=port,
+            username=user,
+            password=passwd,
+        )
+    except Exception as exc:
+        raise ValueError(_sdk_error_message(exc, host, port)) from exc
+
+    windows = _build_hour_windows(since, until)
+    all_rows: list[dict[str, Any]] = []
+
+    for chunk_start, chunk_end in tqdm(
+        windows,
+        desc="fetching",
+        unit="hr",
+        leave=True,
+        bar_format="{desc}: {n_fmt} hours [{elapsed}]",
+    ):
+        earliest = str(int(chunk_start.timestamp()))
+        latest   = str(int(chunk_end.timestamp()))
+        try:
+            job = service.jobs.oneshot(
+                spl,
+                count=0,
+                output_mode="json",
+                earliest_time=earliest,
+                latest_time=latest,
+            )
+        except Exception as exc:
+            raise ValueError(_sdk_error_message(exc, host, port)) from exc
+        chunk = [r for r in splunk_results.JSONResultsReader(job) if isinstance(r, dict)]
+        all_rows.extend(chunk)
+
+    return all_rows, {"units": len(windows), "unit_label": "chunks"}
+
+
+def write(
+    rows: list[dict[str, Any]],
+    outpath: Path,
+    verbose: bool,
+) -> tuple[int, dict[str, Any]]:
+    """Write syslog rows to a flat text file, one line per event.
+
+    Sorts by _time ascending, strips RFC 3164 PRI prefixes, writes non-empty lines.
+
+    Args:
+        rows: Result rows from fetch(), each with _time and _raw fields.
+        outpath: Destination file path.
+        verbose: Reserved for future use.
+
+    Returns:
+        ``(line_count, write_meta)`` where ``write_meta`` carries
+        ``{"bytes": int, "paths": list[Path]}``. Splunk writes a single file —
+        ``paths`` is a one-element list.
+    """
+    rows_sorted = sorted(rows, key=lambda r: r.get("_time", ""))
+    count = 0
+    byte_total = 0
+    try:
+        outpath.parent.mkdir(parents=True, exist_ok=True)
+        with outpath.open("w", encoding="utf-8") as fh:
+            for row in rows_sorted:
+                raw = PRI_RE.sub("", row.get("_raw", "").strip())
+                if raw:
+                    line = raw + "\n"
+                    fh.write(line)
+                    byte_total += len(line.encode("utf-8"))
+                    count += 1
+    except OSError as exc:
+        raise ValueError(f"Could not write export file {outpath}: {exc}") from exc
+    return count, {"bytes": byte_total, "paths": [outpath]}

loghunter/outputs/__init__.py

@@ -0,0 +1 @@
+"""Output format handlers. Each module registers itself via output.register_handler()."""

loghunter/outputs/allowlist.py

@@ -0,0 +1,75 @@
+"""Allowlist export renderer — writes flat allowlist lines to stdout.
+
+Invoked when --export-allowlist is passed. Bypasses the normal output pipeline
+entirely. Output is ready to paste directly into a flat allowlist file.
+
+Format (one line per finding, sorted by score descending):
+  192.0.2.10  192.0.2.1  :22/tcp  # score=0.610 period=60.0s
+"""
+
+from __future__ import annotations
+
+import sys
+
+from loghunter.common.finding import Finding
+
+
+def render(findings: list[Finding]) -> None:
+    """Write flat allowlist lines to stdout, sorted by score descending.
+
+    Each line encodes src_ip, dst_ip, and an optional :port/proto token drawn
+    from finding.evidence. Only findings that carry at least src_ip and dst_ip
+    are emitted; findings from detectors that don't populate those fields are
+    silently skipped.
+
+    Example output:
+      192.0.2.10  192.0.2.1  :22/tcp  # score=0.610 period=60.0s
+    """
+    exportable = [f for f in findings if _has_ip_pair(f)]
+    exportable.sort(key=lambda f: float(f.evidence.get("beacon_score", 0.0)), reverse=True)
+
+    if not exportable:
+        return
+
+    print("# loghunter --export-allowlist — review each entry before merging into allowlist")
+    print()
+
+    for finding in exportable:
+        ev = finding.evidence
+        src = ev.get("src_ip", "")
+        dst = ev.get("dst_ip", "")
+        port_token = _port_token(ev.get("dst_port"), ev.get("proto", ""))
+        score = ev.get("beacon_score", ev.get("score", 0.0))
+        period_str = ev.get("period_str", "")
+
+        parts = [src, dst]
+        if port_token:
+            parts.append(port_token)
+
+        comment = f"# score={float(score):.3f}"
+        if period_str:
+            comment += f" period={period_str}"
+
+        line = "  ".join(parts) + "  " + comment
+        print(line, file=sys.stdout)
+
+
+def _has_ip_pair(finding: Finding) -> bool:
+    ev = finding.evidence
+    return bool(ev.get("src_ip")) and bool(ev.get("dst_ip"))
+
+
+def _port_token(port: int | str | None, proto: str | None) -> str:
+    """Build a :port/proto token. Returns empty string if port is absent or zero."""
+    if port is None:
+        return ""
+    try:
+        port_int = int(port)
+    except (TypeError, ValueError):
+        return ""
+    if port_int == 0:
+        return ""
+    proto = (proto or "").strip()
+    if proto:
+        return f":{port_int}/{proto}"
+    return f":{port_int}"

loghunter/outputs/csv.py

@@ -0,0 +1,70 @@
+"""CSV output handler — flattened findings for spreadsheet import.
+
+Evidence dict is flattened with dot-notation keys (e.g. evidence.score).
+One row per finding. Suitable for Excel, Google Sheets, or pandas downstream.
+"""
+
+from __future__ import annotations
+
+import csv
+import sys
+from typing import Any, TextIO
+
+from loghunter.common.finding import Finding, RunSummary
+from loghunter.common.output import OutputHandler, register_handler
+
+
+class CsvHandler(OutputHandler):
+    """Write findings as CSV to stdout or a file."""
+
+    def __init__(self, stream: TextIO = sys.stdout, verbose_level: int = 0) -> None:
+        self._stream = stream
+        self._verbose_level = verbose_level
+        self._writer: csv.DictWriter | None = None
+        self._rows: list[dict[str, Any]] = []
+
+    def begin(self, run_summary: RunSummary) -> None:
+        """Store run metadata for inclusion in each CSV row."""
+        self._run_summary = run_summary
+
+    def write(self, findings: list[Finding]) -> None:
+        """Write one CSV row per finding."""
+        self._rows.extend(self._flatten_finding(f) for f in findings)
+
+    def end(self) -> None:
+        """Write the CSV header and rows once all evidence keys are known."""
+        base_fields = [
+            "detector",
+            "severity",
+            "title",
+            "description",
+            "ts_generated",
+            "data_window_start",
+            "data_window_end",
+        ]
+        evidence_fields = sorted(
+            {key for row in self._rows for key in row if key.startswith("evidence.")}
+        )
+        fieldnames = base_fields + evidence_fields
+        self._writer = csv.DictWriter(self._stream, fieldnames=fieldnames)
+        self._writer.writeheader()
+        for row in self._rows:
+            self._writer.writerow(row)
+
+    def _flatten_finding(self, finding: Finding) -> dict[str, Any]:
+        """Flatten a Finding into a dict with dot-notation evidence keys."""
+        row: dict[str, Any] = {
+            "detector": finding.detector,
+            "severity": finding.severity.name.lower(),
+            "title": finding.title,
+            "description": finding.description if self._verbose_level >= 1 else "",
+            "ts_generated": finding.ts_generated.isoformat(),
+            "data_window_start": finding.data_window[0].isoformat(),
+            "data_window_end": finding.data_window[1].isoformat(),
+        }
+        for key, value in finding.evidence.items():
+            row[f"evidence.{key}"] = value
+        return row
+
+
+register_handler("csv", CsvHandler)

loghunter/outputs/email.py

@@ -0,0 +1,44 @@
+"""Email output handler — plain text suitable for piping to sendmail. (planned)
+
+Formats findings as a plain-text email body. Handler is dormant — not registered
+in the output registry and has no shipped config section. Wiring up will land a
+new config surface and handler registration.
+"""
+
+from __future__ import annotations
+
+from loghunter.common.finding import Finding, RunSummary
+from loghunter.common.output import OutputHandler, register_handler
+
+
+class EmailHandler(OutputHandler):
+    """Format findings as a plain-text email and send via SMTP."""
+
+    def __init__(
+        self,
+        smtp_host: str = "localhost",
+        smtp_port: int = 25,
+        to: str = "",
+        from_addr: str = "",
+    ) -> None:
+        self._smtp_host = smtp_host
+        self._smtp_port = smtp_port
+        self._to = to
+        self._from = from_addr
+        self._findings: list[Finding] = []
+        self._run_summary: RunSummary | None = None
+
+    def begin(self, run_summary: RunSummary) -> None:
+        """Store run summary for the email subject line."""
+        ...
+
+    def write(self, findings: list[Finding]) -> None:
+        """Accumulate findings for transmission at end()."""
+        ...
+
+    def end(self) -> None:
+        """Compose and send the email via SMTP."""
+        ...
+
+
+register_handler("email", EmailHandler)

loghunter/outputs/html.py

@@ -0,0 +1,99 @@
+"""HTML output handler — self-contained report file for browser viewing.
+
+Produces a single .html file with the same content as text output, formatted
+for readability in a browser. No external dependencies — all styles inline.
+"""
+
+from __future__ import annotations
+
+import html
+from pathlib import Path
+
+from loghunter.common.finding import Finding, RunSummary
+from loghunter.common.output import OutputHandler, register_handler
+
+
+class HtmlHandler(OutputHandler):
+    """Write findings as a self-contained HTML report file."""
+
+    def __init__(self, output_path: Path | None = None, verbose_level: int = 0) -> None:
+        if output_path is None:
+            output_path = Path("loghunter-report.html")
+        self._output_path = output_path
+        self._verbose_level = verbose_level
+        self._findings: list[Finding] = []
+        self._run_summary: RunSummary | None = None
+
+    def begin(self, run_summary: RunSummary) -> None:
+        """Store run summary for the report header."""
+        self._run_summary = run_summary
+
+    def write(self, findings: list[Finding]) -> None:
+        """Accumulate findings for rendering at end()."""
+        self._findings.extend(findings)
+
+    def end(self) -> None:
+        """Render and write the complete HTML file."""
+        self._output_path.parent.mkdir(parents=True, exist_ok=True)
+        self._output_path.write_text(self._render_html(), encoding="utf-8")
+
+    def _render_html(self) -> str:
+        """Produce the full HTML document string."""
+        summary = self._run_summary
+        window = ""
+        records = ""
+        detectors = ""
+        if summary is not None:
+            start, end = summary.data_window
+            window = f"{start:%Y-%m-%d %H:%M} → {end:%Y-%m-%d %H:%M}"
+            records = " · ".join(
+                f"{count:,} {html.escape(name)}"
+                for name, count in summary.record_counts.items()
+            )
+            detectors = " ".join(html.escape(name) for name in summary.detectors_run)
+
+        findings = "\n".join(self._render_finding(f) for f in self._findings)
+        return f"""<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <title>LogHunter report</title>
+  <style>
+    body {{ font: 15px/1.45 -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; margin: 32px; color: #1f2933; }}
+    header {{ border-bottom: 1px solid #c9d1d9; margin-bottom: 24px; padding-bottom: 16px; }}
+    h1 {{ font-size: 24px; margin: 0 0 12px; }}
+    h2 {{ font-size: 18px; margin: 28px 0 10px; }}
+    .meta {{ color: #52606d; }}
+    .finding {{ border-top: 1px solid #e5e8eb; padding: 12px 0; }}
+    .tag {{ font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-weight: 700; }}
+    pre {{ background: #f6f8fa; padding: 12px; overflow-x: auto; }}
+  </style>
+</head>
+<body>
+  <header>
+    <h1>LogHunter report</h1>
+    <div class="meta">Data found: {window}</div>
+    <div class="meta">Records: {records}</div>
+    <div class="meta">Detectors: {detectors}</div>
+  </header>
+  <main>
+    {findings}
+  </main>
+</body>
+</html>
+"""
+
+    def _render_finding(self, finding: Finding) -> str:
+        """Render one finding block."""
+        evidence = html.escape(str(finding.evidence))
+        description = f"<p>{html.escape(finding.description)}</p>" if self._verbose_level >= 1 else ""
+        return (
+            '<section class="finding">'
+            f'<div><span class="tag">{html.escape(str(finding.severity))}</span> '
+            f'{html.escape(finding.detector)} - {html.escape(finding.title)}</div>'
+            f"{description}<pre>{evidence}</pre>"
+            "</section>"
+        )
+
+
+register_handler("html", HtmlHandler)

loghunter/outputs/json.py

@@ -0,0 +1,77 @@
+"""JSON output handler — structured findings, one object per finding.
+
+Suitable for piping to jq or downstream tooling. Outputs a JSON array.
+All datetime fields serialized as ISO 8601 strings.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from typing import Any, TextIO
+
+from loghunter.common.finding import Finding, RunSummary
+from loghunter.common.output import OutputHandler, register_handler
+
+
+class JsonHandler(OutputHandler):
+    """Write findings as a JSON array to stdout or a file."""
+
+    def __init__(self, stream: TextIO = sys.stdout, verbose_level: int = 0) -> None:
+        self._stream = stream
+        self._verbose_level = verbose_level
+        self._findings: list[Finding] = []
+        self._run_summary: RunSummary | None = None
+
+    def begin(self, run_summary: RunSummary) -> None:
+        """Store run summary for inclusion in output."""
+        self._run_summary = run_summary
+
+    def write(self, findings: list[Finding]) -> None:
+        """Accumulate findings for serialization at end()."""
+        self._findings.extend(findings)
+
+    def end(self) -> None:
+        """Serialize all accumulated findings to JSON and write to stream."""
+        payload = {
+            "run_summary": self._run_summary_to_dict(self._run_summary),
+            "findings": [self._finding_to_dict(f) for f in self._findings],
+        }
+        json.dump(payload, self._stream, indent=2, default=str)
+        print(file=self._stream)
+
+    def _finding_to_dict(self, finding: Finding) -> dict[str, Any]:
+        """Convert a Finding to a JSON-serializable dict."""
+        return {
+            "detector": finding.detector,
+            "severity": finding.severity.name.lower(),
+            "severity_tag": str(finding.severity),
+            "title": finding.title,
+            "description": finding.description,
+            "evidence": finding.evidence,
+            "next_steps": finding.next_steps,
+            "ts_generated": finding.ts_generated.isoformat(),
+            "data_window": [
+                finding.data_window[0].isoformat(),
+                finding.data_window[1].isoformat(),
+            ],
+        }
+
+    def _run_summary_to_dict(self, run_summary: RunSummary | None) -> dict[str, Any] | None:
+        """Convert RunSummary to a JSON-serializable dict."""
+        if run_summary is None:
+            return None
+        return {
+            "data_window": [
+                run_summary.data_window[0].isoformat(),
+                run_summary.data_window[1].isoformat(),
+            ],
+            "record_counts": run_summary.record_counts,
+            "data_size_bytes": run_summary.data_size_bytes,
+            "detectors_run": run_summary.detectors_run,
+            "detectors_skipped": run_summary.detectors_skipped,
+            "notes": run_summary.notes,
+        }
+
+
+register_handler("json", JsonHandler)