loghunter-cli 0.1.0.dev0__py3-none-any.whl

loghunter/common/clustering.py

@@ -0,0 +1,326 @@
+"""HDBSCAN backend shim — prefer fast_hdbscan, fall back to stock hdbscan.
+
+The dns detector was originally built and calibrated on stock ``hdbscan``;
+``fast_hdbscan`` was added later as a drop-in accelerator with an identical
+``HDBSCAN(min_cluster_size=, min_samples=)`` plus ``.fit_predict(X)`` API.
+Both produce equivalent findings.
+
+This module resolves which implementation is in use exactly once at import
+time and exposes that class at module level as ``HDBSCAN`` so callers can
+``from loghunter.common.clustering import HDBSCAN`` and construct it
+directly — no factory, no rename. ``ACTIVE_BACKEND`` records which one
+resolved.
+
+Resolution order: ``fast_hdbscan`` first (the accelerator, opt-in via the
+``loghunt[fast]`` extra), then stock ``hdbscan`` (the guaranteed base
+dependency). If neither is importable we raise ``ImportError`` at import
+time rather than letting the failure surface later at a construction site.
+
+Process-isolated entry point
+----------------------------
+
+``fit_predict_interruptible`` is the new shared call site for DNS clustering
+(both Zeek and Pi-hole paths). It runs ``HDBSCAN(...).fit_predict(X)`` in a
+spawned child process so the parent main thread can honour Ctrl-C
+regardless of the native call's GIL state. On ``KeyboardInterrupt`` the
+child is terminated, the queue is drained and closed, and the exception
+re-raises to the caller (the existing ``liveness()`` teardown + the
+``cli.main()`` top-level handler print "Stopped." and exit 130).
+
+For notebook / standalone callers (or any environment where ``spawn`` is
+fragile — Jupyter is the canonical case), the module-level switch
+``_CLUSTERING_ISOLATE_ENABLED`` may be flipped to ``False``; the helper
+then runs in-process via ``_inline_fit_predict`` and preserves today's
+``HDBSCAN(min_cluster_size=, min_samples=).fit_predict(X)`` call verbatim.
+The CLI/runner path inherits the default ON.
+"""
+
+from __future__ import annotations
+
+import multiprocessing
+import queue as _queue
+from typing import Any
+
+import numpy as np
+
+try:
+    from fast_hdbscan import HDBSCAN
+    ACTIVE_BACKEND = "fast_hdbscan"
+except ImportError:
+    try:
+        from hdbscan import HDBSCAN
+        ACTIVE_BACKEND = "hdbscan"
+    except ImportError as e:
+        raise ImportError(
+            "No HDBSCAN backend available — neither 'hdbscan' nor "
+            "'fast_hdbscan' was importable. The 'hdbscan' package is a base "
+            "dependency of loghunt and should always be present; reinstall "
+            "with 'pip install hdbscan', or 'pip install loghunt[fast]' for "
+            "the fast_hdbscan accelerator."
+        ) from e
+
+
+# ── Process-isolation knobs ──────────────────────────────────────────────────
+
+# Quarantine-style switch — mirrors digest/blob.py:_BLOB_DRAIN3_ENABLED.
+# When False, fit_predict_interruptible runs in-process (today's behaviour
+# exactly). The CLI/runner path inherits the default True; notebook /
+# standalone callers can flip it for in-process determinism, or to keep
+# multiprocessing out of a Jupyter kernel where spawn is fragile.
+# DO NOT route this through DetectorContext — detector signatures stay
+# clean; this is environment-shaped, not detector-shaped.
+_CLUSTERING_ISOLATE_ENABLED: bool = True
+
+# Queue polling interval — the main thread wakes this often to check the
+# child's exit status. 100 ms is fast enough that Ctrl-C feels instant
+# without spinning the CPU.
+_POLL_INTERVAL_SEC: float = 0.1
+
+# Bounded join window after SIGTERM before escalating to SIGKILL.
+_TERMINATE_TIMEOUT_SEC: float = 1.0
+
+# Brief join on the NORMAL-return path so a healthy child exit isn't
+# converted into a SIGTERM for no reason. Not used on the interrupt path —
+# the committed interrupt sequence is terminate → join → kill (no grace).
+_GRACEFUL_JOIN_SEC: float = 0.2
+
+
+def _build_clusterer(
+    backend: str, *, min_cluster_size: int, min_samples: int,
+) -> Any:
+    """Construct an HDBSCAN clusterer with backend-conditional kwargs.
+
+    Stock ``hdbscan`` gets ``core_dist_n_jobs=1`` so it spawns no nested
+    multiprocessing pool — our one child is then the only extra process
+    and SIGKILL is clean (no ``resource_tracker`` "leaked semaphore"
+    warning on shutdown). ``fast_hdbscan`` has no ``core_dist_n_jobs``
+    parameter (would TypeError); it uses numba threads anyway, so there
+    are no semaphores to leak.
+
+    Called by ``_cluster_worker`` in the SPAWNED CHILD only. The
+    in-process escape-hatch path (``_inline_fit_predict``) does NOT
+    call this helper — that path preserves today's
+    ``HDBSCAN(min_cluster_size=, min_samples=)`` construction
+    byte-for-byte to avoid drifting the detector's calibration surface.
+    """
+    if backend == "hdbscan":
+        return HDBSCAN(
+            min_cluster_size=min_cluster_size,
+            min_samples=min_samples,
+            core_dist_n_jobs=1,
+        )
+    return HDBSCAN(
+        min_cluster_size=min_cluster_size,
+        min_samples=min_samples,
+    )
+
+
+def _cluster_worker(
+    result_queue: Any,
+    X: "np.ndarray",
+    min_cluster_size: int,
+    min_samples: int,
+    backend: str,
+) -> None:
+    """Module-level worker for spawn pickling.
+
+    Picklable BECAUSE it is module-level — nested functions, lambdas, and
+    closures would not survive ``spawn`` re-import. Constructs an HDBSCAN
+    clusterer via ``_build_clusterer`` (which applies the
+    backend-conditional ``core_dist_n_jobs=1`` for stock hdbscan), calls
+    ``.fit_predict(X)``, and puts ``("ok", labels)`` or
+    ``("error", "<ExcType>: <msg>")`` on the queue.
+
+    Does NOT raise to the parent — all failures become serialised error
+    tuples so the parent path is uniform and arbitrary exception objects
+    never need to round-trip through pickle.
+
+    ``backend`` is passed explicitly (not re-read from ``ACTIVE_BACKEND``
+    in the child) so that test paths exercising backend-conditional
+    behaviour can target a fixed string without monkeypatching across
+    the spawn boundary.
+    """
+    try:
+        clusterer = _build_clusterer(
+            backend,
+            min_cluster_size=min_cluster_size,
+            min_samples=min_samples,
+        )
+        labels = clusterer.fit_predict(X)
+        result_queue.put(("ok", labels))
+    except Exception as exc:  # noqa: BLE001 — serialised, not raised
+        result_queue.put(("error", f"{type(exc).__name__}: {exc}"))
+
+
+# Indirection seam for tests: rebind this to one of the module-level test
+# helpers in tests/test_clustering_interruptible.py to exercise specific
+# child-process behaviours (block, raise, die without queueing) without
+# closures crossing the spawn boundary.
+_WORKER_TARGET = _cluster_worker
+
+
+def _await_child_result(
+    result_queue: Any, child: "multiprocessing.Process",
+) -> tuple:
+    """Poll the result queue + child liveness until one of them yields.
+
+    Does NOT call an indefinite ``queue.get()`` — that would hang
+    forever if the child segfaults / OOMs / exits without putting a
+    result. Instead polls with ``queue.get(timeout=_POLL_INTERVAL_SEC)``
+    and, between polls, checks ``child.is_alive()`` / ``child.exitcode``.
+
+    On a dead child without a queued result, raises ``RuntimeError`` —
+    a normal exception that the existing CLI ``ValueError``/``OSError``
+    arms can surface. Never returns ``None``.
+
+    ``KeyboardInterrupt`` propagates naturally: ``queue.get`` is a
+    Python-level wait, so SIGINT delivered to the main thread is
+    raised out of this function and the caller's interrupt branch
+    handles cleanup.
+    """
+    while True:
+        try:
+            return result_queue.get(timeout=_POLL_INTERVAL_SEC)
+        except _queue.Empty:
+            pass
+        if not child.is_alive():
+            exitcode = child.exitcode
+            raise RuntimeError(
+                "DNS clustering worker died "
+                f"(exitcode={exitcode}) without returning a result. "
+                "The clustering library may have crashed; try the "
+                "alternate backend (pip install 'loghunt[fast]' or "
+                "pip install hdbscan)."
+            )
+
+
+def _drain_and_close_queue(result_queue: Any) -> None:
+    """Drain pending items, then close and join the feeder thread.
+
+    Called from BOTH the normal-return and interrupt cleanup paths,
+    always AFTER the child is no longer alive. ``close()`` + ``join_thread()``
+    together prevent the ``multiprocessing.resource_tracker`` "leaked
+    semaphore" warning at process shutdown; ``close()`` alone is not
+    enough (the feeder thread may still hold a reference). Draining
+    pending items first stops ``close()`` from blocking the feeder on
+    unsent data.
+    """
+    try:
+        while True:
+            result_queue.get_nowait()
+    except _queue.Empty:
+        pass
+    result_queue.close()
+    result_queue.join_thread()
+
+
+def _inline_fit_predict(
+    X: "np.ndarray", min_cluster_size: int, min_samples: int,
+) -> "np.ndarray":
+    """In-process escape hatch — preserves today's calibration surface
+    exactly. NOT routed through ``_build_clusterer`` on purpose: the
+    backend-conditional ``core_dist_n_jobs=1`` is a child-process
+    resource-tracker concern, not a calibration choice we want to drift
+    into notebook / standalone callers.
+    """
+    clusterer = HDBSCAN(
+        min_cluster_size=min_cluster_size, min_samples=min_samples,
+    )
+    return clusterer.fit_predict(X)
+
+
+def fit_predict_interruptible(
+    X: "np.ndarray", *, min_cluster_size: int, min_samples: int,
+) -> "np.ndarray":
+    """HDBSCAN ``.fit_predict(X)`` with Ctrl-C honoured on a long compute.
+
+    Replaces the inline ``clusterer = HDBSCAN(...); labels =
+    clusterer.fit_predict(X)`` shape at the two DNS call sites. Return
+    contract is identical (label int array, shape ``(len(X),)``); the
+    detector logic above/below is unchanged.
+
+    When ``_CLUSTERING_ISOLATE_ENABLED`` is True (the CLI/runner default),
+    runs the compute in a spawned child process so SIGINT delivered to
+    the parent main thread is honoured regardless of the native call's
+    GIL state. On ``KeyboardInterrupt`` the child is terminated and the
+    exception re-raises to the caller (``liveness()`` teardown +
+    ``cli.main()``'s top-level handler print "Stopped." and exit 130).
+
+    When False (notebook / standalone escape hatch), runs in-process via
+    ``_inline_fit_predict`` and preserves today's behaviour byte-for-byte.
+
+    Raises:
+        ValueError: when the child reports a clustering failure
+            (degenerate input, etc.) — preserves the detector contract
+            that a clustering failure surfaces as a normal exception.
+        RuntimeError: when the child dies without putting a result
+            (segfault, OOM kill, etc.) — never silently hangs.
+        KeyboardInterrupt: re-raised after child termination so the
+            existing ``liveness()`` + ``cli.main()`` machinery handles
+            teardown.
+    """
+    if not _CLUSTERING_ISOLATE_ENABLED:
+        return _inline_fit_predict(X, min_cluster_size, min_samples)
+
+    ctx = multiprocessing.get_context("spawn")
+    result_queue = ctx.Queue()
+    child = ctx.Process(
+        target=_WORKER_TARGET,
+        args=(result_queue, X, min_cluster_size, min_samples, ACTIVE_BACKEND),
+    )
+    child.start()
+
+    # Universal cleanup discipline. The try body runs the await + an
+    # optional graceful join on the normal-return path; the finally
+    # ALWAYS runs the committed terminate → kill → drain/close → close
+    # sequence regardless of how the body exited. This covers four
+    # cases with one structure:
+    #
+    #   - Normal return: await yields, graceful join runs in the body,
+    #     finally's terminate/kill are no-ops (child already exited),
+    #     queue is drained+closed, child handle closed.
+    #   - KeyboardInterrupt: await raises, body skips straight to
+    #     finally. Graceful join is NEVER reached on this path — the
+    #     operator wants the helper out NOW — so the finally's
+    #     terminate fires immediately. This matches the committed
+    #     interrupt sequence (terminate → join → kill → cleanup →
+    #     re-raise) exactly: the re-raise is the implicit post-finally
+    #     propagation.
+    #   - RuntimeError (dead child without result): await raises,
+    #     finally runs. Child is already dead so terminate/kill are
+    #     no-ops, but queue drain/close + child.close still run —
+    #     preventing the resource_tracker leak on the exact abnormal-
+    #     death path the helper exists to handle.
+    #   - Any future non-KBI exception from the await path: same
+    #     guarantee as the RuntimeError case. If the child happens to
+    #     still be alive, the finally's terminate/kill takes it down
+    #     before draining the queue (closing the queue with a live
+    #     child would hang the feeder thread).
+    try:
+        result = _await_child_result(result_queue, child)
+        # NORMAL-RETURN PATH continues here. Brief join so a healthy
+        # exit isn't converted to SIGTERM by the finally below; if
+        # the child stalls past the grace window, finally's
+        # terminate/kill takes over.
+        if child.is_alive():
+            child.join(_GRACEFUL_JOIN_SEC)
+    finally:
+        if child.is_alive():
+            child.terminate()
+            child.join(_TERMINATE_TIMEOUT_SEC)
+        if child.is_alive():
+            child.kill()
+            child.join()
+        _drain_and_close_queue(result_queue)
+        child.close()
+
+    if result[0] == "ok":
+        return result[1]
+    raise ValueError(f"DNS clustering failed in worker: {result[1]}")
+
+
+__all__ = [
+    "HDBSCAN",
+    "ACTIVE_BACKEND",
+    "fit_predict_interruptible",
+]

loghunter/common/config.py

@@ -0,0 +1,221 @@
+"""Config loading with precedence chain: CLI arg > user > system.
+
+Precedence (highest to lowest):
+  1. Explicit --config=FILE argument
+  2. ~/.loghunter/config.toml  (user default)
+  3. /etc/loghunter/config.toml  (system-wide)
+
+When no config file is found, returns a deep copy of built-in defaults — no exception raised.
+"""
+
+from __future__ import annotations
+
+import copy
+import tomllib
+from datetime import timedelta
+from pathlib import Path
+from typing import Any
+
+
+class ConfigError(Exception):
+    """Raised for config problems that the user needs to act on."""
+
+
+def parse_window_span(spec: str | None) -> timedelta | None:
+    """Parse a default_window config value into a timedelta.
+
+    Returns None for: None, "", "all" (case-insensitive) — meaning "no default".
+    Accepts: "Nd" (days), "Nh" (hours) where N is a positive integer.
+    Raises ConfigError for any other value — silent fallback hides real config bugs.
+    """
+    if spec is None:
+        return None
+    s = str(spec).strip()
+    if s == "" or s.lower() == "all":
+        return None
+    try:
+        if s.endswith("d"):
+            n = int(s[:-1])
+            if n > 0:
+                return timedelta(days=n)
+        elif s.endswith("h"):
+            n = int(s[:-1])
+            if n > 0:
+                return timedelta(hours=n)
+    except ValueError:
+        pass
+    raise ConfigError(
+        f"default_window={spec!r} is not a valid duration. "
+        f"Use 'Nd' (days), 'Nh' (hours), '' or 'all' to disable."
+    )
+
+
+_DEFAULTS: dict[str, Any] = {
+    "loghunter": {
+        # LH_ROOT — base for RELATIVE paths in config-file values. Empty = use
+        # CWD for relative paths. Absolute and ~-anchored paths ignore it.
+        # Env override: LOGHUNTER_ROOT (env wins over config).
+        "root": "~/.loghunter",
+        "detect": "all",
+        # Conventional source locations; tried out-of-box. pihole/cloudtrail
+        # stay None (opt-in — no missing-file warning when absent).
+        "zeek_dir": "/var/log/zeek",
+        "syslog_dir": "/var/log",
+        "pihole_dir": None,
+        "cloudtrail_dir": None,
+        # Internal networks for traffic-direction classification. Topology
+        # fact, not detector tuning. RFC1918 default; override only if your
+        # internal address plan differs.
+        "home_net": ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"],
+        # Where exporters write pulled logs. Backends and per-query stanzas
+        # may override per the precedence cascade.
+        # Trailing slash communicates directory intent to be_like_water —
+        # without it, a non-existent path would be interpreted as a FILE.
+        "export_dir": "exports/",
+        # report_dir intentionally OMITTED — no shipped default. Setting it is
+        # an explicit opt-in to file-mode analyze output. Bare analyze prints
+        # to stdout when report_dir is unset and --out is not passed.
+        "output_format": "text",
+        "default_window": "1d",
+        "warn_above": 5_000_000,
+        # Per-detector total row cap for TEXT output only (json/csv/html
+        # render everything — machine formats must not lose data). The cap
+        # is a running budget across the detector's subsections in declared
+        # order; the disclosure line reports rendered-vs-total. 0 = unlimited.
+        "max_findings_per_detector": 100,
+    },
+    "detectors": {},
+    "allowlist": {
+        "domain_patterns": [
+            "~/.loghunter/allowlist.d/domains_user.txt",
+        ],
+        "connection_rules": [
+            "~/.loghunter/allowlist.d/connections.txt",
+        ],
+        "allowlist_dir": "~/.loghunter/allowlist.d/",
+    },
+    "export": {
+        "splunk": {
+            "host": "",
+            "port": 8089,
+            "username": "",
+            "password": "",
+        },
+        # cloudtrail exporter — boto3 pull from S3, writes CloudTrail JSON locally.
+        # path is the s3:// URL to the CloudTrail tree; egress_warn_gb is the
+        # cost guard threshold. Activation is a non-empty path.
+        "cloudtrail": {
+            "path": "",
+            "egress_warn_gb": 5.0,
+        },
+    },
+}
+
+SEARCH_PATHS: list[Path] = [
+    Path("~/.loghunter/config.toml").expanduser(),
+    Path("/etc/loghunter/config.toml"),
+]
+
+
+def load(config_file: str | Path | None = None) -> dict[str, Any]:
+    """Load config from the precedence chain and return the merged config dict.
+
+    If config_file is given, it is used directly; raises ConfigError if missing.
+    If no config file is found in the search path, returns built-in defaults cleanly.
+    """
+    if config_file is not None:
+        path = Path(config_file)
+        if not path.exists():
+            raise ConfigError(
+                f"Config file not found: {path}\n"
+                f"Check the path or run 'loghunter init' to create a config."
+            )
+        config = _load_file(path)
+    else:
+        found = _find_config_file()
+        if found is None:
+            config = copy.deepcopy(_DEFAULTS)
+        else:
+            config = _load_file(found)
+
+    # Validate default_window eagerly so typos in user config fail at load time —
+    # not lazily during the run, where bounded paths would never notice.
+    parse_window_span(config.get("loghunter", {}).get("default_window"))
+    return config
+
+
+def default_allowlist_paths() -> dict[str, Any]:
+    """Return a deep copy of ``_DEFAULTS["allowlist"]`` — the single source of
+    truth for fallback paths when an allowlist config key is absent.
+
+    Used by ``common/allowlist.py:build_matcher`` when a raw / notebook config
+    arrives without ``domain_patterns``, ``connection_rules``, or
+    ``allowlist_dir`` set (the ``cfg.load`` deep-merge would otherwise have
+    supplied them from ``_DEFAULTS``).
+    """
+    return copy.deepcopy(_DEFAULTS["allowlist"])
+
+
+def get_detector_config(
+    config: dict[str, Any],
+    detector_name: str,
+    detector_defaults: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Return the merged config for a specific detector.
+
+    File config wins over detector_defaults, which win over nothing.
+    """
+    base = copy.deepcopy(detector_defaults or {})
+    file_section = config.get("detectors", {}).get(detector_name, {})
+    return _deep_merge(base, file_section)
+
+
+def _find_config_file() -> Path | None:
+    """Walk SEARCH_PATHS and return the first existing file."""
+    for path in SEARCH_PATHS:
+        if path.exists():
+            return path
+    return None
+
+
+def _load_file(path: Path) -> dict[str, Any]:
+    """Parse a TOML config file and deep-merge it over built-in defaults.
+
+    Attaches a ``__user_set__`` sidecar to the returned merged dict: a mapping
+    from top-level section name to the set of key names the operator declared
+    in that section. This is provenance metadata for runner-level disclosures
+    (e.g. "default RFC1918 vs. operator-declared home_net") — a value-only
+    check cannot distinguish a defaulted value from a user-declared value that
+    happens to equal the default. The "no config file found" path in load()
+    skips _load_file entirely; absence of the sidecar is correctly read as
+    "no user declarations".
+    """
+    try:
+        with path.open("rb") as fh:
+            user_config = tomllib.load(fh)
+    except tomllib.TOMLDecodeError as exc:
+        raise ConfigError(
+            f"Config file parse error in {path}:\n  {exc}\n"
+            f"Check the file for TOML syntax errors."
+        ) from exc
+
+    merged = _deep_merge(copy.deepcopy(_DEFAULTS), user_config)
+    merged["__user_set__"] = {
+        section: set(content.keys()) if isinstance(content, dict) else set()
+        for section, content in user_config.items()
+    }
+    return merged
+
+
+def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
+    """Recursively merge override into base, returning a new dict.
+
+    Scalars and lists in override replace those in base. Dicts are merged recursively.
+    """
+    result = dict(base)
+    for key, val in override.items():
+        if key in result and isinstance(result[key], dict) and isinstance(val, dict):
+            result[key] = _deep_merge(result[key], val)
+        else:
+            result[key] = copy.deepcopy(val)
+    return result