logometer 0.1.0__py3-none-any.whl

logometer/explain.py

@@ -0,0 +1,185 @@
+"""
+Optional "--explain" support: sends an anomalous window's lines to an
+LLM and asks for a one-sentence plain-English summary of the likely
+cause.
+
+Providers: Anthropic, OpenAI, and OpenRouter (OpenAI-compatible, gives
+access to many vendors' models with one key).
+
+Deliberately built on stdlib `urllib` rather than the `anthropic` or
+`openai` SDKs, so `--explain` needs nothing beyond an API key — no
+extra pip install, no risk of drifting out of sync with an SDK's API
+surface. This is a deviation from the original plan (which sketched
+"Anthropic or OpenAI SDK"); seemed like the better trade for a tool
+whose whole pitch is "works with nothing installed."
+
+This module must NEVER raise anything except ExplainError — callers
+(cli.py) rely on that to fail soft (print a note, keep tailing) rather
+than crashing the whole run because an anomaly happened to coincide
+with a flaky network.
+"""
+from __future__ import annotations
+
+import json
+import os
+import urllib.error
+import urllib.request
+
+_ANTHROPIC_URL = "https://api.anthropic.com/v1/messages"
+_ANTHROPIC_MODEL = "claude-haiku-4-5-20251001"  # fast + cheap, all we need for a one-liner
+
+_OPENAI_URL = "https://api.openai.com/v1/chat/completions"
+_OPENAI_MODEL = "gpt-4o-mini"
+
+# OpenRouter speaks the OpenAI chat-completions format, so it shares
+# that code path; model ids are namespaced, e.g. "openai/gpt-4o-mini".
+_OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
+_OPENROUTER_MODEL = "anthropic/claude-haiku-4.5"
+
+_TIMEOUT_SECONDS = 15
+_MAX_LINES_SENT = 30  # cap what we send: a window shouldn't need more context than this to explain
+
+_SYSTEM_PROMPT = (
+    "You are looking at a burst of anomalous lines from an application log. "
+    "In ONE short sentence (under 25 words), state the most likely underlying cause. "
+    "Be concrete and specific to what's in the lines. No preamble, no hedging, no markdown."
+)
+
+
+class ExplainError(Exception):
+    """Raised for any failure in getting an explanation — missing API
+    key, network failure, bad response, unknown provider. Callers
+    should catch this and degrade gracefully rather than crash."""
+
+
+def _build_prompt(lines: list[str]) -> str:
+    sample = lines[:_MAX_LINES_SENT]
+    joined = "\n".join(sample)
+    if len(lines) > _MAX_LINES_SENT:
+        joined += f"\n... ({len(lines) - _MAX_LINES_SENT} more similar lines omitted)"
+    return f"Log lines:\n{joined}"
+
+
+def _call_anthropic(lines: list[str], model: str) -> str:
+    api_key = os.environ.get("ANTHROPIC_API_KEY")
+    if not api_key:
+        raise ExplainError(
+            "ANTHROPIC_API_KEY is not set. Set it to enable --explain, "
+            "e.g. export ANTHROPIC_API_KEY=your-key-here"
+        )
+
+    body = json.dumps({
+        "model": model,
+        "max_tokens": 100,
+        "system": _SYSTEM_PROMPT,
+        "messages": [{"role": "user", "content": _build_prompt(lines)}],
+    }).encode("utf-8")
+
+    request = urllib.request.Request(
+        _ANTHROPIC_URL,
+        data=body,
+        method="POST",
+        headers={
+            "Content-Type": "application/json",
+            "x-api-key": api_key,
+            "anthropic-version": "2023-06-01",
+        },
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=_TIMEOUT_SECONDS) as response:
+            payload = json.loads(response.read().decode("utf-8"))
+    except urllib.error.HTTPError as e:
+        detail = e.read().decode("utf-8", errors="replace")[:300]
+        raise ExplainError(f"Anthropic API returned {e.code}: {detail}") from e
+    except urllib.error.URLError as e:
+        raise ExplainError(f"could not reach Anthropic API: {e.reason}") from e
+    except TimeoutError as e:
+        raise ExplainError("Anthropic API request timed out") from e
+
+    try:
+        blocks = payload["content"]
+        text = "".join(b.get("text", "") for b in blocks if b.get("type") == "text").strip()
+    except (KeyError, TypeError) as e:
+        raise ExplainError(f"unexpected response shape from Anthropic API: {payload!r}") from e
+
+    if not text:
+        raise ExplainError("Anthropic API returned an empty explanation")
+    return text
+
+
+def _call_openai_compatible(
+    lines: list[str], model: str, *, url: str, key_env: str, provider: str, label: str
+) -> str:
+    api_key = os.environ.get(key_env)
+    if not api_key:
+        raise ExplainError(
+            f"{key_env} is not set. Set it to enable --explain with --explain-provider {provider}, "
+            f"e.g. export {key_env}=your-key-here"
+        )
+
+    body = json.dumps({
+        "model": model,
+        "max_tokens": 100,
+        "messages": [
+            {"role": "system", "content": _SYSTEM_PROMPT},
+            {"role": "user", "content": _build_prompt(lines)},
+        ],
+    }).encode("utf-8")
+
+    request = urllib.request.Request(
+        url,
+        data=body,
+        method="POST",
+        headers={
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {api_key}",
+        },
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=_TIMEOUT_SECONDS) as response:
+            payload = json.loads(response.read().decode("utf-8"))
+    except urllib.error.HTTPError as e:
+        detail = e.read().decode("utf-8", errors="replace")[:300]
+        raise ExplainError(f"{label} API returned {e.code}: {detail}") from e
+    except urllib.error.URLError as e:
+        raise ExplainError(f"could not reach {label} API: {e.reason}") from e
+    except TimeoutError as e:
+        raise ExplainError(f"{label} API request timed out") from e
+
+    try:
+        text = payload["choices"][0]["message"]["content"].strip()
+    except (KeyError, IndexError, TypeError, AttributeError) as e:
+        raise ExplainError(f"unexpected response shape from {label} API: {payload!r}") from e
+
+    if not text:
+        raise ExplainError(f"{label} API returned an empty explanation")
+    return text
+
+
+def _call_openai(lines: list[str], model: str) -> str:
+    return _call_openai_compatible(
+        lines, model, url=_OPENAI_URL, key_env="OPENAI_API_KEY", provider="openai", label="OpenAI"
+    )
+
+
+def _call_openrouter(lines: list[str], model: str) -> str:
+    return _call_openai_compatible(
+        lines, model, url=_OPENROUTER_URL, key_env="OPENROUTER_API_KEY", provider="openrouter", label="OpenRouter"
+    )
+
+
+_PROVIDERS = {
+    "anthropic": (_call_anthropic, _ANTHROPIC_MODEL),
+    "openai": (_call_openai, _OPENAI_MODEL),
+    "openrouter": (_call_openrouter, _OPENROUTER_MODEL),
+}
+
+
+def explain_window(lines: list[str], provider: str = "anthropic", model: str | None = None) -> str:
+    """Get a one-sentence explanation for a burst of anomalous log
+    lines. Raises ExplainError on any failure — callers should catch
+    this and continue without the explanation rather than crash."""
+    if provider not in _PROVIDERS:
+        raise ExplainError(f"unknown --explain-provider {provider!r}, expected one of {list(_PROVIDERS)}")
+    fn, default_model = _PROVIDERS[provider]
+    return fn(lines, model or default_model)

logometer/pretty.py

@@ -0,0 +1,52 @@
+"""
+Nicer terminal rendering using `rich`, used automatically when it's
+installed and we're writing to a real terminal. Falls back to the
+plain ANSI formatting in cli.py otherwise — this module is never
+required, only opportunistic.
+"""
+from __future__ import annotations
+
+try:
+    from rich.console import Console
+    from rich.panel import Panel
+    from rich.text import Text
+    HAS_RICH = True
+except ImportError:
+    HAS_RICH = False
+
+
+def render_window(window, verdict, explanation: str | None, console: "Console") -> None:
+    """Render one window's verdict with rich. Only call this if
+    HAS_RICH is True."""
+    label = f"{window.start_label} \u2013 {window.end_label}"
+
+    if not verdict.is_anomaly:
+        text = Text(f"  [{label}]  ok   errors: {window.error_count}   baseline: ~{verdict.baseline_mean:.1f}")
+        text.stylize("dim")
+        console.print(text)
+        return
+
+    body = Text()
+    body.append(f"errors: {window.error_count}   ", style="bold")
+    if window.warn_count:
+        # a first-seen WARN shape can flag a window with zero errors
+        body.append(f"warns: {window.warn_count}   ", style="bold")
+    body.append(f"baseline: ~{verdict.baseline_mean:.1f}")
+    if verdict.error_score:
+        body.append(f"   score: {verdict.error_score:.1f}x", style="bold red" if verdict.error_score >= 3 else "yellow")
+
+    if verdict.new_shapes:
+        body.append("\n")
+        levels = {l.shape: l.level for l in window.lines if l.shape}
+        for shape in verdict.new_shapes:
+            kind = "warning" if levels.get(shape) == "WARN" else "error"
+            body.append(f"\n  New {kind} signature: ", style="yellow")
+            body.append(shape, style="italic")
+    elif not explanation:
+        body.append("\n\n  (error rate spike \u2014 no brand-new error signature)", style="dim")
+
+    if explanation:
+        body.append("\n\n  ")
+        body.append(explanation, style="cyan")
+
+    console.print(Panel(body, title=f"\u26a0 ANOMALY  [{label}]", border_style="red", expand=False))

logometer/timeparse.py

@@ -0,0 +1,58 @@
+"""
+Best-effort extraction of a timestamp from the start of a log line.
+
+We only need this to decide how to bucket lines into windows. If a
+line's timestamp can't be parsed, the caller falls back to a
+line-count-based window instead of a time-based one — see windower.py.
+"""
+from __future__ import annotations
+
+import re
+from datetime import datetime
+from typing import Optional
+
+# ISO 8601, e.g. "2026-09-17T12:03:10.123Z" or "2026-09-17 12:03:10"
+_ISO_RE = re.compile(
+    r"(\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?)(Z|[+-]\d{2}:?\d{2})?"
+)
+
+# Syslog-style, e.g. "Sep 17 12:03:10"
+_SYSLOG_RE = re.compile(
+    r"\b([A-Z][a-z]{2}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2})\b"
+)
+
+_SYSLOG_MONTHS = {
+    "Jan": 1, "Feb": 2, "Mar": 3, "Apr": 4, "May": 5, "Jun": 6,
+    "Jul": 7, "Aug": 8, "Sep": 9, "Oct": 10, "Nov": 11, "Dec": 12,
+}
+
+
+def parse_timestamp(line: str, assumed_year: Optional[int] = None) -> Optional[datetime]:
+    """Best-effort ISO or syslog timestamp extraction from a log line.
+    Returns None when no recognizable time is found (caller may use line-based windows)."""
+    m = _ISO_RE.search(line)
+    if m:
+        raw = m.group(1)
+        try:
+            # normalize a couple of common variants Python's fromisoformat
+            # is picky about
+            raw = raw.replace(" ", "T") if "T" not in raw else raw
+            return datetime.fromisoformat(raw)
+        except ValueError:
+            pass
+
+    m = _SYSLOG_RE.search(line)
+    if m:
+        try:
+            month_str, day_str, time_str = m.group(1).split(None, 2)
+            # handle "Sep  1 12:03:10" (double space) already collapsed by split
+            month = _SYSLOG_MONTHS.get(month_str)
+            if month is None:
+                return None
+            year = assumed_year or datetime.now().year
+            hh, mm, ss = (int(x) for x in time_str.split(":"))
+            return datetime(year, month, int(day_str), hh, mm, ss)
+        except (ValueError, IndexError):
+            return None
+
+    return None

logometer/windower.py

@@ -0,0 +1,247 @@
+"""
+Groups a stream of classified log lines into fixed-size windows.
+
+Two modes, chosen automatically:
+  - "timed": if timestamps can be parsed from the log lines, windows are
+    bucketed by wall-clock/log-clock time (--window seconds).
+  - "count": if no timestamps are found (first N lines all fail to
+    parse), windows are bucketed by a fixed number of lines instead.
+    This keeps the tool useful on logs with no recognizable timestamp
+    format, at the cost of windows not corresponding to a fixed time span.
+"""
+from __future__ import annotations
+
+import math
+import time
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Iterator, Optional
+
+from .classifier import ClassifiedLine, classify_line
+from .timeparse import parse_timestamp
+
+# how many leading lines we sample to decide timed vs. count mode
+_PROBE_LINES = 20
+# fraction of probe lines that must have a parseable timestamp to use timed mode
+_PROBE_THRESHOLD = 0.5
+# fallback window size (in lines) when no timestamps are parseable
+_DEFAULT_LINES_PER_WINDOW = 50
+
+
+@dataclass
+class Window:
+    index: int
+    start_label: str  # human-readable label for the window (time range or line range)
+    end_label: str
+    lines: list[ClassifiedLine] = field(default_factory=list)
+
+    @property
+    def error_count(self) -> int:
+        """Number of lines in this window classified as ERROR.
+        Fed to RollingBaseline as the primary spike metric."""
+        return sum(1 for l in self.lines if l.level == "ERROR")
+
+    @property
+    def warn_count(self) -> int:
+        """Number of lines in this window classified as WARN.
+        Included in JSON output; not used for baseline spike scoring."""
+        return sum(1 for l in self.lines if l.level == "WARN")
+
+    @property
+    def shapes(self) -> set[str]:
+        """Distinct error/warn fingerprints present in this window.
+        Compared against AnomalyDetector._seen_shapes for new-signature anomalies."""
+        return {l.shape for l in self.lines if l.shape}
+
+
+class WindowAggregator:
+    """Feed classified lines in; get completed Window objects out.
+
+    Usage:
+        agg = WindowAggregator(window_seconds=10)
+        for line in source:
+            for window in agg.feed(line):
+                handle(window)
+        for window in agg.flush():
+            handle(window)
+    """
+
+    def __init__(self, window_seconds: float = 10.0, lines_per_window: int = _DEFAULT_LINES_PER_WINDOW):
+        """Create an aggregator; timed vs count mode is chosen after probing first lines.
+        window_seconds applies in timed mode; lines_per_window in count mode."""
+        self.window_seconds = window_seconds
+        self.lines_per_window = lines_per_window
+
+        self._mode: Optional[str] = None  # "timed" or "count", decided after probing
+        self._probe_buffer: list[str] = []
+        self._probe_timestamps: list[Optional[datetime]] = []
+        self._probe_started_at: Optional[float] = None
+
+        self._current: Optional[Window] = None
+        self._current_bucket_key = None
+        self._window_index = 0
+        self._first_timestamp: Optional[datetime] = None
+        # real-time clock (not log time) for the in-progress window, so
+        # tick() can close it on schedule when the log goes silent
+        self._current_opened_at: Optional[float] = None
+
+    # -- mode detection -----------------------------------------------------
+
+    def _decide_mode(self) -> None:
+        """Pick timed windows if enough probe lines had timestamps, else count-based.
+        Sets _first_timestamp when entering timed mode."""
+        parseable = sum(1 for t in self._probe_timestamps if t is not None)
+        if self._probe_timestamps and parseable / len(self._probe_timestamps) >= _PROBE_THRESHOLD:
+            self._mode = "timed"
+            for t in self._probe_timestamps:
+                if t is not None:
+                    self._first_timestamp = t
+                    break
+        else:
+            self._mode = "count"
+
+    # -- public API -----------------------------------------------------
+
+    def feed(self, raw_line: str) -> Iterator[Window]:
+        """Ingest one raw log line; yield any windows that closed (often zero or one).
+        Buffers an initial probe batch before choosing timed vs count mode."""
+        raw_line = raw_line.rstrip("\n")
+        if not raw_line:
+            return
+
+        if self._mode is None:
+            if self._probe_started_at is None:
+                self._probe_started_at = time.monotonic()
+            self._probe_buffer.append(raw_line)
+            self._probe_timestamps.append(parse_timestamp(raw_line))
+            if len(self._probe_buffer) < _PROBE_LINES:
+                return  # still probing, hold lines until mode is decided
+            self._decide_mode()
+            # replay the buffered probe lines now that we know the mode
+            buffered = self._probe_buffer
+            self._probe_buffer = []
+            for buffered_line in buffered:
+                yield from self._feed_decided(buffered_line)
+            return
+
+        yield from self._feed_decided(raw_line)
+
+    def _feed_decided(self, raw_line: str) -> Iterator[Window]:
+        """Classify a line, assign it to the current bucket, yield completed windows.
+        Requires _mode to already be timed or count."""
+        classified = classify_line(raw_line)
+
+        if self._mode == "timed":
+            ts = parse_timestamp(raw_line) or self._last_seen_timestamp()
+            bucket_key = math.floor((ts - self._first_timestamp).total_seconds() / self.window_seconds)
+            self._last_ts = ts
+        else:
+            bucket_key = None  # computed below from line count
+
+        if self._current is None:
+            self._current = self._new_window(bucket_key)
+            self._current_bucket_key = bucket_key
+
+        if self._mode == "count":
+            if len(self._current.lines) >= self.lines_per_window:
+                yield self._current
+                self._window_index += 1
+                self._current = self._new_window(None)
+        else:
+            if bucket_key != self._current_bucket_key:
+                yield self._current
+                self._window_index += 1
+                self._current = self._new_window(bucket_key)
+                self._current_bucket_key = bucket_key
+
+        self._current.lines.append(classified)
+
+    def _last_seen_timestamp(self) -> datetime:
+        """Fallback clock when a line in timed mode has no parseable timestamp.
+        Reuses last seen time, else log start, else wall clock."""
+        return getattr(self, "_last_ts", self._first_timestamp or datetime.now())
+
+    def _new_window(self, bucket_key) -> Window:
+        """Allocate the next window with human-readable start/end labels.
+        Timed labels are HH:MM:SS ranges; count mode uses line number ranges."""
+        if self._mode == "timed" and bucket_key is not None:
+            start = self._first_timestamp.timestamp() + bucket_key * self.window_seconds
+            end = start + self.window_seconds
+            start_label = datetime.fromtimestamp(start).strftime("%H:%M:%S")
+            end_label = datetime.fromtimestamp(end).strftime("%H:%M:%S")
+        else:
+            start_n = self._window_index * self.lines_per_window + 1
+            end_n = start_n + self.lines_per_window - 1
+            start_label = f"line {start_n}"
+            end_label = f"line {end_n}"
+        self._current_opened_at = time.monotonic()
+        return Window(index=self._window_index, start_label=start_label, end_label=end_label)
+
+    def tick(self, now: Optional[float] = None) -> list[Window]:
+        """Close anything whose time is up, without needing a new line.
+
+        Feeding alone only closes a window when the *next* line arrives
+        in a later bucket. On a live tail that means a service which
+        errors and then dies never gets reported at all: the burst that
+        matters most stays buffered forever, because nothing follows it.
+        Callers poll this while idle so windows close on schedule.
+
+        Two things can be overdue:
+          - the in-progress window, once `window_seconds` of real time
+            have passed since it opened
+          - the probe batch itself, on a log too quiet to produce 20
+            lines — otherwise a low-volume service reports nothing ever
+
+        Real (monotonic) time is used deliberately, not log time: a log
+        whose clock is skewed or in another timezone would otherwise
+        close every window the moment it went idle.
+        """
+        now = time.monotonic() if now is None else now
+        closed: list[Window] = []
+
+        if self._mode is None:
+            waited = self._probe_started_at is not None and now - self._probe_started_at >= self.window_seconds
+            if not (self._probe_buffer and waited):
+                return []
+            self._decide_mode()
+            buffered = self._probe_buffer
+            self._probe_buffer = []
+            for buffered_line in buffered:
+                closed.extend(self._feed_decided(buffered_line))
+
+        if (
+            self._current is not None
+            and self._current.lines
+            and self._current_opened_at is not None
+            and now - self._current_opened_at >= self.window_seconds
+        ):
+            closed.append(self._current)
+            self._current = None
+            self._current_opened_at = None
+            self._window_index += 1
+
+        return closed
+
+    def flush(self) -> list[Window]:
+        """Return every window still pending at EOF, oldest first.
+
+        Usually that's just the one in-progress window. But on a file
+        shorter than the probe batch, mode is decided here and the
+        buffered lines are replayed now — which can close several
+        windows at once. All of them must come back: an earlier version
+        kept only the last one, so a short log silently lost every
+        window but its final one (and with them the baseline history
+        the detector needs to flag anything at all).
+        """
+        pending: list[Window] = []
+        if self._mode is None and self._probe_buffer:
+            # never reached probe threshold (short file) — decide now
+            self._decide_mode()
+            buffered = self._probe_buffer
+            self._probe_buffer = []
+            for buffered_line in buffered:
+                pending.extend(self._feed_decided(buffered_line))
+        if self._current and self._current.lines:
+            pending.append(self._current)
+            self._current = None
+        return pending