loghunter-cli 0.1.0.dev0__py3-none-any.whl

loghunter/common/display.py

@@ -0,0 +1,323 @@
+"""Shared display constants and helpers for human-facing terminal output.
+
+The ``liveness`` context manager is the shared primitive for ROADMAP-12 TTY
+progress narration. It draws an indeterminate spinner on stderr for opaque
+blocking phases when stderr is a tty, and seals a permanent one-line record
+on the way out (visible on tty AND non-tty — only the live animation is
+tty-gated). Countable phases stay on ``tqdm``; ``liveness`` is for the cases
+where there is no natural tick.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import threading
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Any, Iterable, Iterator
+
+from tqdm import tqdm
+
+TEXT_RULE_WIDTH = 80
+TEXT_RULE = "─" * TEXT_RULE_WIDTH
+TEXT_RULE_DOUBLE = "═" * TEXT_RULE_WIDTH
+
+# Spinner frames cycle in this exact order. ASCII only — no unicode-width
+# surprises in clearing math.
+_SPINNER_FRAMES = ("|", "/", "-", "\\")
+# Per-frame interval. 120ms sits in the middle of the 100-150ms band that
+# reads as "alive" without thrashing the terminal.
+_SPINNER_INTERVAL_S = 0.12
+
+
+# ── Method-chrome color seam (text handler only) ────────────────────────────
+#
+# Minimal: a single SGR constant for the method-glow paint, a TTY/NO_COLOR
+# gate, and one paint() helper. Not a general terminal-capability layer; the
+# only consumer is the text handler's Detectors: line. Rebound for retuning
+# the glow in a single place — Dave will experiment live.
+_METHOD_SGR = "\x1b[96;1m"  # bright-cyan + bold
+_RESET = "\x1b[0m"
+
+
+def _stream_isatty(stream: Any) -> bool:
+    """Raw TTY probe. No color-policy coupling.
+
+    Shared by ``_color_enabled``, ``_LivenessHandle``, and ``progress``. Color
+    layers ``NO_COLOR``/``TERM=dumb`` on top; liveness and progress gate on
+    TTY only — a color preference is not a progress preference.
+    """
+    isatty = getattr(stream, "isatty", lambda: False)
+    try:
+        return bool(isatty())
+    except Exception:
+        return False
+
+
+def _color_enabled(stream: Any) -> bool:
+    """True when the stream is a real TTY and color is not opted out.
+
+    Honors the NO_COLOR ambient convention and the legacy TERM=dumb signal.
+    File streams (--out / report_dir) and pipes are not TTYs and therefore
+    plain — automatic, no extra wiring at call sites.
+    """
+    if not _stream_isatty(stream):
+        return False
+    if os.environ.get("NO_COLOR") is not None:
+        return False
+    if os.environ.get("TERM") == "dumb":
+        return False
+    return True
+
+
+def paint(text: str, *, stream: Any) -> str:
+    """Wrap ``text`` in the method SGR when ``stream`` admits color.
+
+    No-op on non-TTYs, on NO_COLOR-set environments, and on TERM=dumb. The
+    single SGR constant ``_METHOD_SGR`` is the one place to retune the glow.
+    """
+    return f"{_METHOD_SGR}{text}{_RESET}" if _color_enabled(stream) else text
+
+
+def human_bytes(n: float) -> str:
+    """Human-readable -h-style byte size.
+
+    Consolidated from outputs/text.py's prior ``_format_bytes``. Consumers
+    today: the digest/blob renderers in text.py and the exporter narration
+    in W4. Digest ``conn.py`` keeps its deliberate local helper (kept-local
+    note in DESIGN.md) — do not repoint that one.
+    """
+    if n < 1024:
+        return f"{int(n)} B"
+    if n < 1024 ** 2:
+        return f"{n / 1024:.1f} KB"
+    if n < 1024 ** 3:
+        return f"{n / (1024 ** 2):.1f} MB"
+    if n < 1024 ** 4:
+        return f"{n / (1024 ** 3):.1f} GB"
+    return f"{n / (1024 ** 4):.1f} TB"
+
+
+def compact_home(path: "str | Path") -> str:
+    """Return ``path`` as a string with the user's home prefix replaced by ``~``.
+
+    Pure display helper for exporter narration and similar surfaces. Operates
+    on the STRING form so a trailing slash is preserved (callers shouldn't
+    have to special-case that). Returns ``path`` unchanged when it doesn't
+    fall under ``$HOME`` or when ``HOME`` is unset.
+    """
+    text = str(path)
+    home = os.path.expanduser("~")
+    if not home or home == "~":
+        return text
+    if text == home:
+        return "~"
+    prefix = home if home.endswith(os.sep) else home + os.sep
+    if text.startswith(prefix):
+        return "~" + os.sep + text[len(prefix):]
+    return text
+
+
+def progress(
+    iterable: Iterable[Any],
+    *,
+    desc: str,
+    show_progress: bool = True,
+    unit: str = " lines",
+    total: int | None = None,
+    stream: Any = None,
+) -> Iterator[Any]:
+    """TTY-aware tqdm wrapper for loader read loops.
+
+    Returns a counting GENERATOR on a TTY when ``show_progress`` is True;
+    otherwise returns the bare iterable (``tqdm`` is NEVER constructed). Gate
+    is raw isatty + the explicit ``show_progress`` flag; color policy
+    (``NO_COLOR``/``TERM=dumb``) is NOT consulted — a color preference is not
+    a progress preference.
+
+    The TTY branch constructs the tqdm WITHOUT an iterable and drives it via
+    ``bar.update(1)`` from a generator. This is what makes the count survive
+    PARSER RE-ITERATION: parsers that sniff-then-resume (Zeek
+    ``itertools.chain(prefix, line_iter)``; CloudTrail's second loop) call
+    ``iter()`` on the returned object a second time — for a generator,
+    ``iter(gen) is gen``, so the same generator (and its counter) continues,
+    whereas a bare tqdm's own ``__iter__`` would be orphaned (the
+    ``loaded X: 0.00 lines`` bug).
+
+    The pinned ``bar_format`` reproduces the long-standing NDJSON loader bar
+    byte-for-byte when ``unit=" lines"``; ``unit`` is parameterized so future
+    non-line-oriented callers can be honest about what they count.
+    """
+    if stream is None:
+        stream = sys.stderr
+    if not show_progress or not _stream_isatty(stream):
+        return iter(iterable)
+    bar = tqdm(
+        desc=desc,
+        unit=unit,
+        unit_scale=True,
+        leave=True,
+        mininterval=0.5,
+        total=total,
+        file=stream,
+        bar_format=f"{{desc}}: {{n_fmt}}{unit} [{{elapsed}}]",
+    )
+
+    def _counting() -> Iterator[Any]:
+        try:
+            for item in iterable:
+                bar.update(1)
+                yield item
+        finally:
+            bar.close()
+
+    return _counting()
+
+
+class _LivenessHandle:
+    """Handle returned from a ``liveness`` context manager.
+
+    Owns the spinner thread, the captured stderr stream, and the bookkeeping
+    needed to keep ``seal()`` and ``__exit__`` honest about whether the
+    spinner actually drew anything.
+    """
+
+    def __init__(self, label: str, delay: float) -> None:
+        self._label = label
+        self._delay = delay
+        # Capture the stream at construction so a single
+        # monkeypatch.setattr(sys, "stderr", fake) before __enter__ is
+        # observed for the whole lifecycle, and so a later redirect of
+        # sys.stderr does not steal our writes.
+        self._stream = sys.stderr
+        self._isatty = _stream_isatty(self._stream)
+        self._stop = threading.Event()
+        self._thread: threading.Thread | None = None
+        # _drew flips True inside the spinner thread immediately before its
+        # first frame write. seal() / __exit__ only emit a clearing sequence
+        # when _drew is True — a phase that seals before the spinner ever
+        # drew prints exactly the sealed line, no \r flicker.
+        self._drew = False
+        self._sealed = False
+        # Guard concurrent writes between the spinner thread and seal/exit.
+        self._lock = threading.Lock()
+
+    # ── lifecycle ───────────────────────────────────────────────────────────
+
+    def _start(self) -> None:
+        if not self._isatty:
+            return
+        self._thread = threading.Thread(target=self._spin, daemon=True)
+        self._thread.start()
+
+    def _spin(self) -> None:
+        # Wait out the initial delay. If sealed during the delay, leave
+        # without ever writing — this is the seal-before-delay invariant.
+        if self._stop.wait(self._delay):
+            return
+        i = 0
+        frames = _SPINNER_FRAMES
+        while not self._stop.is_set():
+            with self._lock:
+                if self._stop.is_set():
+                    return
+                # Flip _drew before the first write so the buffer state
+                # and the flag agree from the same critical section.
+                self._drew = True
+                try:
+                    self._stream.write(f"\r{frames[i % len(frames)]} {self._label}")
+                    self._stream.flush()
+                except Exception:
+                    return
+            i += 1
+            if self._stop.wait(_SPINNER_INTERVAL_S):
+                return
+
+    def _clear_line(self) -> None:
+        # Pad-erase wide enough to cover one frame char + space + label.
+        width = 2 + len(self._label)
+        try:
+            self._stream.write("\r" + (" " * width) + "\r")
+            self._stream.flush()
+        except Exception:
+            pass
+
+    def seal(self, text: str) -> None:
+        """Commit a permanent one-line record and stop the spinner.
+
+        Idempotent — a second seal is a no-op. On a tty, clears the spinner
+        line first (only if the spinner actually drew); otherwise writes the
+        record straight to the stream. The sealed record is the only stderr
+        artifact that promises "this phase finished cleanly."
+        """
+        with self._lock:
+            if self._sealed:
+                return
+            self._sealed = True
+            self._stop.set()
+        if self._thread is not None:
+            self._thread.join()
+        with self._lock:
+            if self._isatty and self._drew:
+                self._clear_line()
+            try:
+                self._stream.write(f"{text}\n")
+                self._stream.flush()
+            except Exception:
+                pass
+
+    def _teardown(self, had_exception: bool) -> None:
+        """Single teardown path called from __exit__.
+
+        Stops the spinner thread, then clears the partial spinner line if
+        the spinner drew anything. Never writes a sealed record — that is
+        seal()'s job, and a body that did not call seal() (whether by
+        exception or by choice) leaves no record.
+        """
+        with self._lock:
+            already_sealed = self._sealed
+            self._stop.set()
+        if self._thread is not None:
+            self._thread.join()
+        if already_sealed:
+            # seal() already cleared and wrote; nothing more to do.
+            return
+        with self._lock:
+            if self._isatty and self._drew:
+                self._clear_line()
+        # had_exception is informational — same teardown either way once
+        # we know seal() did not fire. A partial spinner gets cleared; no
+        # record is written. The exception (if any) propagates from
+        # __exit__'s caller.
+
+
+@contextmanager
+def liveness(label: str, delay: float = 0.0) -> Iterator[_LivenessHandle]:
+    """Indeterminate-spinner liveness for an opaque blocking phase.
+
+    Usage::
+
+        with liveness("running beacon") as ln:
+            findings = run_beacon(ctx)
+            ln.seal(f"beacon: {len(findings)} findings")
+
+    On a tty, draws a single-line spinner ``"<frame> <label>"`` on stderr
+    after ``delay`` seconds (so fast phases never flicker). On non-tty,
+    draws nothing. ``ln.seal(text)`` commits a permanent record line.
+
+    If the body raises (including KeyboardInterrupt — Ctrl-C during the
+    phase), the spinner line is cleared, no sealed record is written, and
+    the exception propagates. This is what lets the runner's top-level
+    Ctrl-C handler print "Stopped." without a false-success seal landing
+    just before it.
+    """
+    handle = _LivenessHandle(label, delay)
+    handle._start()
+    try:
+        yield handle
+    except BaseException:
+        handle._teardown(had_exception=True)
+        raise
+    else:
+        handle._teardown(had_exception=False)

loghunter/common/errors.py

@@ -0,0 +1,45 @@
+"""Shared exception types used across runner, exporters, and CLI.
+
+ExportAborted lives here (not under exporters/) so that runner.py can raise it
+without creating a runner → exporter dependency. The CLI catches it once and
+translates to a clean exit 0.
+"""
+
+from __future__ import annotations
+
+
+class ExportAborted(Exception):
+    """Raised when the operator declines an advisory confirmation prompt.
+
+    Used by the runner's large-dataset prompt and by exporter backends'
+    cost-prompts (e.g. CloudTrail's S3 egress guard). Caught by cli.main()
+    and translated to a clean exit 0 with the message. Not a ValueError —
+    distinct from the user-facing error path.
+    """
+
+
+class DigestEmpty(Exception):
+    """Raised by run_digest when a RECOGNIZED schema loads to an empty frame.
+
+    Not an error — the file was understood, it simply had no parseable
+    records (e.g. a Zeek conn.log with header rows but zero data rows).
+    Callers catch this and narrate it without rendering a card.
+
+    Explicitly NOT a subclass of ValueError: catch-arms in cli.py that
+    handle real per-path failures (corrupt gzip, parser errors) MUST NOT
+    consume DigestEmpty, which is a control signal carrying a successful
+    "the file was understood and contained nothing to render" outcome.
+
+    basename: filename when the digest source was a file (sniff-driven
+    fan-out, single-file source_dir); directory name when the source was
+    a configured directory (bare-config branch). The stderr narration
+    "recognized as <schema> but no parseable records" reads correctly
+    in both cases.
+    """
+
+    def __init__(self, basename: str, schema: str) -> None:
+        super().__init__(
+            f"recognized {basename} as {schema} but no parseable records"
+        )
+        self.basename = basename
+        self.schema = schema

loghunter/common/finding.py

@@ -0,0 +1,239 @@
+"""Finding dataclass and Severity enum — the shared contract between detectors and outputs.
+
+Detectors produce list[Finding]. Output handlers consume list[Finding].
+Nothing else crosses that boundary.
+
+DigestCard and DigestSlot are peer types to Finding/RunSummary — used by the
+digest verb to render an orient-before-the-hunt card. They carry no severity,
+no evidence, no next_steps; digest never produces a verdict.
+"""
+
+from __future__ import annotations
+
+import copy
+import enum
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+    from loghunter.common.allowlist import AllowlistMatcher
+
+
+class Severity(enum.Enum):
+    """Detection severity levels, rendered as bracketed tags in text output."""
+
+    HIGH = "H"
+    MEDIUM = "M"
+    LOW = "L"
+    INFO = "I"
+
+    def __str__(self) -> str:
+        return f"[{self.value}]"
+
+    def __lt__(self, other: "Severity") -> bool:
+        _order = [Severity.HIGH, Severity.MEDIUM, Severity.LOW, Severity.INFO]
+        return _order.index(self) < _order.index(other)
+
+
+@dataclass
+class Finding:
+    """A single detection result produced by a detector.
+
+    Detectors return list[Finding]. Output handlers render them.
+    The evidence dict is detector-specific — no fixed schema.
+    description and next_steps are only shown in verbose output.
+    """
+
+    detector: str
+    severity: Severity
+    title: str
+    description: str
+    evidence: dict[str, Any]
+    next_steps: list[str]
+    ts_generated: datetime
+    data_window: tuple[datetime, datetime]
+
+
+@dataclass(frozen=True)
+class MethodTag:
+    """Per-detector method label, surfaced in the run-summary banner.
+
+    ``named=True`` marks a published algorithm (FFT, drain3, fast-HDBSCAN) —
+    rendered with parentheses and painted by the text handler. ``named=False``
+    marks an honest house badge (pattern, heuristics, statistical) — rendered
+    in plain brackets, never painted. The parens-vs-brackets carry 100% of
+    the meaning; color is enhancement only.
+    """
+
+    label: str
+    named: bool
+
+
+@dataclass
+class RunSummary:
+    """Metadata about a loghunter run, printed before analysis begins and passed to output handlers."""
+
+    data_window: tuple[datetime, datetime]
+    record_counts: dict[str, int]
+    data_size_bytes: int
+    detectors_run: list[str]
+    detectors_skipped: dict[str, str]  # name → reason
+    notes: list[str] = field(default_factory=list)
+    data_sources: list[str] = field(default_factory=list)
+    detector_methods: dict[str, "MethodTag | None"] = field(default_factory=dict)
+    # The window the operator asked for (default-window spec, explicit since/until
+    # span, or since→now), used by the text handler's data-found underfill
+    # parenthetical. None = unconstrained (--all / until-only / bounded full-load).
+    requested_span: timedelta | None = None
+
+
+@dataclass
+class DigestSlot:
+    """One row in a DigestCard's fields block.
+
+    Bi-state:
+
+    - SPEAKING: cells (pre-formatted column strings) AND
+      entity/magnitude/ratio (raw values) are populated together. Cells and
+      raw values come from the same source numbers; keeping the raw values
+      lets insight selection sort by salience without parsing "3.7x" back
+      out of a display string.
+
+    - NON-SPEAKING (applicable but nothing notable, or feed cannot compute):
+      all four value fields are None. The renderer never sees these slots
+      — the summariser filters them out before handing fields to the card.
+    """
+
+    label: str                       # "conn-share", "fan-out", ...
+    statistic: str                   # "cliff" | "tail" | "rate" | "share" | "dist"
+    cells: list[str] | None = None
+    entity: str | None = None
+    magnitude: float | None = None
+    ratio: float | None = None
+
+
+@dataclass
+class DigestCard:
+    """A digest's per-schema rendered body. Peer to Finding, not a subclass.
+
+    Carries the spine-derived ambient facts (window, record count, histogram,
+    bytes) plus the summariser-derived ``zone1_extras`` ambient block,
+    ``insights`` (prose sentences mechanically derived from speaking gated
+    slots), and ``fields`` (the display-ready, already-filtered speaking
+    non-insight slots). Selection happens in the summariser; the renderer
+    is dumb.
+
+    ``data_window`` may be ``(None, None)`` when timestamps in the loaded
+    frame are unparseable below the confidence floor; the renderer renders
+    identity line 2 as a bare ``—`` and the histogram line as
+    ``(timeline unavailable)``.
+
+    ``timeline_unavailable`` is the explicit sentinel for the histogram-
+    suppression path. Without it, an empty ``histogram_counts`` could also
+    mean "no events in window" (the valid empty-frame case) — a renderer
+    looking only at counts cannot disambiguate.
+    """
+
+    schema: str
+    source_name: str                  # identity-line-1; basename of source file or dir
+    data_window: tuple[datetime | None, datetime | None]
+    record_count: int
+    histogram_counts: list[int]
+    histogram_unit: str               # "hr" | "day"
+    histogram_peak: int
+    zone1_extras: list[tuple[str, str]] = field(default_factory=list)
+    insights: list[str] = field(default_factory=list)
+    fields: list[DigestSlot] = field(default_factory=list)
+    data_size_bytes: int = 0
+    timeline_unavailable: bool = False
+
+
+@dataclass
+class BlobCard:
+    """Unrecognized-source panel for the blob digest path. Peer to DigestCard,
+    not a subclass. Carries the description-of-bytes-as-bytes panel the blob
+    renderer needs — no slots, no histogram, no data_window — by design.
+
+    The blob path describes bytes and never extracts fields. No timestamp is
+    read, no schema is assumed. ``line_length_shape`` is exactly ``"uniform"``
+    or ``"varied"`` when set. The char-class fractions are computed over RAW
+    sampled BYTES (before UTF-8 decode) so binary-looking input shows up as
+    low ``printable_pct`` rather than being masked by ``errors="replace"``.
+
+    Vanish-don't-dash: optional fields default to None when their slot does
+    not apply (binary terminal magic, drain3 dormant, freeform-no-structure
+    template floor). The renderer omits the row entirely — no "-", no
+    placeholder. Required fields are sample-derived facts that always exist
+    for any input the profiler can read.
+
+    O(sample) rail: every field is computed from the bounded sample. The only
+    whole-file fact is ``byte_size`` (a stat). ``sampled_line_count`` is the
+    line count over the sample, NEVER a whole-file total.
+    """
+
+    # always present — sample-derived facts that exist for any input
+    source_name: str                     # identity-line-1; basename of the source path
+    byte_size: int                       # on-disk size from stat (compressed for .gz)
+    sampled_line_count: int
+    sample_read_count: int               # 1 head + K seeks (plain); 1 (compressed head-only)
+    is_compressed: bool
+    printable_pct: float
+    nonprintable_pct: float
+    utf8_clean: bool                     # strict-decode probe over the sample
+
+    # identification — exactly one of file_type_guess or shape_guess set
+    file_type_guess: str | None = None    # terminal magic label (e.g. "PNG image")
+    file_type_magic: bytes | None = None  # matched magic bytes for the Bytes row repr
+    shape_guess: str | None = None        # text-shape cascade result
+
+    # text slots — None on binary terminal hit
+    mean_line_length: float | None = None
+    median_line_length: float | None = None
+    line_length_p95: int | None = None
+    max_line_length: int | None = None
+    line_length_stdev: float | None = None
+    line_length_shape: str | None = None  # "uniform" | "varied" | None
+
+    top_tokens: list[tuple[str, int]] | None = None
+
+    # JSON shape-guess only — first-seen union of top-level object keys
+    # across the sample. None on binary, on non-JSON text, and on top-
+    # level-array/scalar JSON (no object keys to list). When set, the
+    # renderer emits a `fields:` row in place of `tokens:` — names-no-
+    # values, structurally describing the shape one rung deeper than the
+    # `shape: JSON` label.
+    json_field_names: list[str] | None = None
+
+    # templates — None on binary / freeform-no-structure / drain3 dormant
+    distinct_templates: int | None = None
+    top_template_coverage_pct: float | None = None
+    top_template_n: int | None = None
+    singleton_template_count: int | None = None
+
+
+@dataclass
+class DetectorContext:
+    """Everything a detector needs to do its job.
+
+    The framework constructs this and passes it to each detector's run() function.
+    Detectors never open files, read config, or format output directly.
+
+    home_net is run/environment metadata (operator-declared internal networks
+    for traffic-direction classification) — peer to data_window and data_sources,
+    not detector tuning. Empty list means "not supplied"; detectors that need
+    direction classification may apply a sensible fallback in that case.
+
+    Verbosity is intentionally absent: the result set is verbosity-invariant
+    by construction (W6). Level-aware filtering happens at the text handler,
+    not in detector ``run()``.
+    """
+
+    logs: dict[str, "pd.DataFrame"]
+    config: dict[str, Any]
+    allowlist: "AllowlistMatcher"
+    data_window: tuple[datetime, datetime]
+    data_sources: list[str] = field(default_factory=list)
+    home_net: list[str] = field(default_factory=list)