agentmetry 0.4.0__py3-none-any.whl

agentmetry/core/audit/detection/engine.py

@@ -0,0 +1,78 @@
+"""Detection engine — orders a session's events and runs the rule registry."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+
+from .models import SEVERITY_RANK, Detection
+from .rules import HOST_REGISTRY, REGISTRY
+from .yaml_rules import build_yaml_rules
+
+_EPOCH = datetime.min.replace(tzinfo=timezone.utc)
+
+
+def _session_rules():
+    return [*REGISTRY, *build_yaml_rules()]
+
+
+def _event_ts(event: dict[str, Any]) -> datetime:
+    ts = event.get("timestamp_utc")
+    if not isinstance(ts, str):
+        return _EPOCH
+    try:
+        dt = datetime.fromisoformat(ts.replace("Z", "+00:00"))
+        return dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt
+    except ValueError:
+        return _EPOCH
+
+
+def _sorted(events: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Time order, falling back to the order the caller supplied.
+
+    The tie-break used to end with `event_id`, which is a random UUID. Every
+    sequence rule in this package asks "did A happen before B", so on a
+    timestamp tie the answer was decided by a coin flip.
+
+    Ties are not rare. Windows clock granularity is around 15 ms, so two tool
+    calls from one agent turn routinely share a timestamp; this surfaced as a
+    Windows-only CI failure where `credential-read-then-cloud-api` fired or did
+    not depending on which UUID sorted first. The README claims ordering is
+    enforced by position rather than co-occurrence, and a random tie-break
+    quietly made that untrue.
+
+    `sorted` is stable, so omitting the UUID preserves the input order on ties.
+    Callers hand us events straight from the trail, which returns them ordered
+    by `timestamp_utc` then insert id, so input order *is* arrival order: the
+    best available evidence of what actually happened first.
+    """
+    def _key(event: dict[str, Any]) -> tuple[datetime, int]:
+        seq = event.get("seq")
+        seq_int = seq if isinstance(seq, int) else 0
+        return (_event_ts(event), seq_int)
+
+    return sorted(events, key=_key)
+
+
+def run_detections(events: list[dict[str, Any]]) -> list[Detection]:
+    """Run every rule over one session's events, ranked most-severe first.
+
+    `events` should be the events for a single correlation_id; ordering is
+    handled here so callers can pass them straight from the trail.
+    """
+    ordered = _sorted(events)
+    detections: list[Detection] = []
+    for rule in _session_rules():
+        detections.extend(rule(ordered))
+    detections.sort(key=lambda d: (SEVERITY_RANK.get(d.severity, 99), d.first_seen_utc))
+    return detections
+
+
+def run_host_detections(events: list[dict[str, Any]]) -> list[Detection]:
+    """Run host-scoped rules over events aggregated by host_id."""
+    ordered = _sorted(events)
+    detections: list[Detection] = []
+    for rule in HOST_REGISTRY:
+        detections.extend(rule(ordered))
+    detections.sort(key=lambda d: (SEVERITY_RANK.get(d.severity, 99), d.first_seen_utc))
+    return detections

agentmetry/core/audit/detection/live.py

@@ -0,0 +1,127 @@
+"""Live detection — correlate as events arrive, not only when someone asks.
+
+The rule engine is a pure function over a session's events. That is fine for a
+forensic query, but a detection nobody sees is not a control: until this module
+existed, a `credential-exfil` finding only appeared if an operator happened to
+open that session in the dashboard. Nothing streamed, nothing reached a SIEM,
+nothing alerted.
+
+This keeps a bounded window of each live session, re-runs the rules on every
+ingested event, and returns only detections that are *new* for that session —
+so a firing rule is emitted once, as a canonical event, down the same sinks as
+everything else.
+
+Live state (event windows + emitted rule IDs) is checkpointed in SQLite so an
+orchestrator restart does not re-alert or lose context for active sessions.
+The JSONL trail remains authoritative; `GET /audit/detections/{id}` still
+recomputes from it.
+"""
+
+from __future__ import annotations
+
+import uuid
+from typing import Any
+
+from agentmetry.core.audit.identity import identity_fields
+
+from .engine import run_detections, run_host_detections
+from .live_store import get_live_store
+from .models import Detection
+
+def reset_live_state() -> None:
+    """Test helper — clear live detection checkpoint."""
+    get_live_store().clear_all()
+
+
+def observe(canonical: dict[str, Any]) -> list[Detection]:
+    """Record an event and return detections that are NEW for its session.
+
+    Does NOT checkpoint here. The caller must call `mark_detection_emitted` only
+    after the detection is durably stored and forwarded, so a failed emit lets
+    the rule fire again on the next session event instead of being lost.
+    """
+    corr = str(canonical.get("correlation_id") or "")
+    if not corr:
+        return []
+
+    store = get_live_store()
+    events = store.append_event(corr, canonical)
+
+    fresh: list[Detection] = []
+    seen_in_batch: set[str] = set()
+    for detection in run_detections(events):
+        # Dedup within this call (a rule may return several findings) without
+        # persisting — persistence is the caller's job, post-emit.
+        if detection.rule_id in seen_in_batch or store.is_emitted(corr, detection.rule_id):
+            continue
+        seen_in_batch.add(detection.rule_id)
+        fresh.append(detection)
+    return fresh
+
+
+def observe_host(canonical: dict[str, Any]) -> list[Detection]:
+    """Record an event in the host-level window and return new host-scoped detections."""
+    host_id = str(canonical.get("host_id") or "")
+    if not host_id:
+        return []
+
+    store = get_live_store()
+    events = store.append_host_event(host_id, canonical)
+
+    fresh: list[Detection] = []
+    seen_in_batch: set[str] = set()
+    for detection in run_host_detections(events):
+        if detection.rule_id in seen_in_batch or store.is_host_emitted(host_id, detection.rule_id):
+            continue
+        seen_in_batch.add(detection.rule_id)
+        fresh.append(detection)
+    return fresh
+
+
+def mark_host_detection_emitted(host_id: str, rule_id: str, emitted_at: str = "") -> None:
+    """Checkpoint a host-level detection as emitted."""
+    if not host_id or not rule_id:
+        return
+    get_live_store().mark_host_emitted(host_id, rule_id, emitted_at=emitted_at)
+
+
+def mark_detection_emitted(correlation_id: str, rule_id: str, emitted_at: str = "") -> None:
+    """Checkpoint a detection as emitted for its session.
+
+    Call ONLY after the detection event is durably stored and forwarded. Marking
+    is idempotent (INSERT OR IGNORE), so a re-fired detection that finally
+    succeeds does not double-alert.
+    """
+    if not correlation_id or not rule_id:
+        return
+    get_live_store().mark_emitted(correlation_id, rule_id, emitted_at=emitted_at)
+
+
+def build_detection_event(detection: Detection, source_event: dict[str, Any]) -> dict[str, Any]:
+    """Wrap a Detection as a canonical event so it flows to every sink.
+
+    `action.outcome` carries the severity: a SIEM can alert on
+    `action.type:detection AND action.outcome:critical` without understanding
+    Agentmetry's rule vocabulary.
+    """
+    source = source_event.get("source")
+    agent = source_event.get("agent")
+    return {
+        "schema_version": source_event.get("schema_version", "1.1.0"),
+        "event_id": str(uuid.uuid4()),
+        "session_id": source_event.get("session_id", ""),
+        "correlation_id": detection.correlation_id or source_event.get("correlation_id", ""),
+        "timestamp_utc": source_event.get("timestamp_utc", ""),
+        **identity_fields(),
+        "source_topic": f"detection/{detection.rule_id}",
+        "source": source if isinstance(source, dict) else {"tier": "detection", "app": "agentmetry"},
+        "initiator": source_event.get("initiator", {}),
+        "actor": source_event.get("actor", {}),
+        "action": {
+            "type": "detection",
+            "outcome": detection.severity,
+            "reason": detection.summary,
+        },
+        "agent": agent if isinstance(agent, dict) else {"name": "agentmetry"},
+        "detection": detection.as_dict(),
+    }

agentmetry/core/audit/detection/live_store.py

@@ -0,0 +1,355 @@
+"""SQLite checkpoint for live detection — survives orchestrator restarts.
+
+Persists per-session event windows (bounded) and emitted rule IDs so a restart
+does not re-fire detections or lose correlation context for active sessions.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import sqlite3
+import threading
+import time
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+_MAX_SESSIONS = 256
+_MAX_EVENTS_PER_SESSION = 500
+
+# How long a host-level detection stays suppressed after firing. Session marks
+# are permanent (a session ends); host marks are not (a host does not), so
+# without this a single firing silenced the rule on that machine forever.
+HOST_EMIT_TTL_SECONDS = 6 * 3600
+
+
+def _is_expired(emitted_at: str, *, ttl_seconds: int = HOST_EMIT_TTL_SECONDS) -> bool:
+    """True when a host checkpoint is old enough to re-arm.
+
+    An unparsable timestamp counts as expired: re-alerting is a smaller failure
+    than a rule that is silently off.
+    """
+    try:
+        ts = datetime.fromisoformat(emitted_at.replace("Z", "+00:00"))
+    except (ValueError, AttributeError):
+        return True
+    if ts.tzinfo is None:
+        ts = ts.replace(tzinfo=timezone.utc)
+    return ts < datetime.now(timezone.utc) - timedelta(seconds=ttl_seconds)
+
+_SCHEMA_SQL = """
+CREATE TABLE IF NOT EXISTS live_session_meta (
+    correlation_id TEXT PRIMARY KEY,
+    last_touch     REAL NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS live_events (
+    correlation_id TEXT NOT NULL,
+    seq            INTEGER NOT NULL,
+    event_json     TEXT NOT NULL,
+    PRIMARY KEY (correlation_id, seq)
+);
+
+CREATE INDEX IF NOT EXISTS idx_live_events_corr ON live_events(correlation_id);
+
+CREATE TABLE IF NOT EXISTS live_emitted (
+    correlation_id TEXT NOT NULL,
+    rule_id        TEXT NOT NULL,
+    emitted_at     TEXT NOT NULL DEFAULT '',
+    PRIMARY KEY (correlation_id, rule_id)
+);
+
+CREATE TABLE IF NOT EXISTS live_host_meta (
+    host_id     TEXT PRIMARY KEY,
+    last_touch  REAL NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS live_host_events (
+    host_id     TEXT NOT NULL,
+    seq         INTEGER NOT NULL,
+    event_json  TEXT NOT NULL,
+    PRIMARY KEY (host_id, seq)
+);
+
+CREATE INDEX IF NOT EXISTS idx_live_host_events_host ON live_host_events(host_id);
+
+CREATE TABLE IF NOT EXISTS live_host_emitted (
+    host_id   TEXT NOT NULL,
+    rule_id   TEXT NOT NULL,
+    emitted_at TEXT NOT NULL DEFAULT '',
+    PRIMARY KEY (host_id, rule_id)
+);
+"""
+
+
+class LiveDetectionStore:
+    """Thread-safe SQLite store for live detection state."""
+
+    def __init__(self, db_path: Path | str) -> None:
+        self._db_path = str(db_path)
+        Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
+        self._local = threading.local()
+        self._lock = threading.Lock()
+        self._init_schema()
+
+    def _get_conn(self) -> sqlite3.Connection:
+        conn = getattr(self._local, "conn", None)
+        if conn is None:
+            conn = sqlite3.connect(self._db_path, timeout=10)
+            conn.execute("PRAGMA journal_mode=WAL")
+            conn.execute("PRAGMA synchronous=NORMAL")
+            conn.execute("PRAGMA busy_timeout=5000")
+            self._local.conn = conn
+        return conn
+
+    def _init_schema(self) -> None:
+        conn = self._get_conn()
+        conn.executescript(_SCHEMA_SQL)
+        conn.commit()
+
+    def clear_all(self) -> None:
+        """Test helper — wipe all live detection state."""
+        with self._lock:
+            conn = self._get_conn()
+            conn.execute("DELETE FROM live_events")
+            conn.execute("DELETE FROM live_emitted")
+            conn.execute("DELETE FROM live_session_meta")
+            conn.execute("DELETE FROM live_host_events")
+            conn.execute("DELETE FROM live_host_emitted")
+            conn.execute("DELETE FROM live_host_meta")
+            conn.commit()
+
+    def close(self) -> None:
+        conn = getattr(self._local, "conn", None)
+        if conn is not None:
+            conn.close()
+            self._local.conn = None
+
+    def append_event(self, correlation_id: str, event: dict[str, Any]) -> list[dict[str, Any]]:
+        """Append an event and return the bounded session window."""
+        with self._lock:
+            conn = self._get_conn()
+            # Wall clock, not monotonic: last_touch is persisted, and monotonic
+            # resets on reboot — pre-reboot sessions would outrank new ones and
+            # LRU eviction would evict the newest sessions first.
+            now = time.time()
+            conn.execute(
+                """
+                INSERT INTO live_session_meta (correlation_id, last_touch)
+                VALUES (?, ?)
+                ON CONFLICT(correlation_id) DO UPDATE SET last_touch = excluded.last_touch
+                """,
+                (correlation_id, now),
+            )
+            row = conn.execute(
+                "SELECT COALESCE(MAX(seq), 0) FROM live_events WHERE correlation_id = ?",
+                (correlation_id,),
+            ).fetchone()
+            next_seq = int(row[0]) + 1
+            conn.execute(
+                "INSERT INTO live_events (correlation_id, seq, event_json) VALUES (?, ?, ?)",
+                (correlation_id, next_seq, json.dumps(event, separators=(",", ":"), sort_keys=True)),
+            )
+            min_seq = max(1, next_seq - _MAX_EVENTS_PER_SESSION + 1)
+            conn.execute(
+                "DELETE FROM live_events WHERE correlation_id = ? AND seq < ?",
+                (correlation_id, min_seq),
+            )
+            self._evict_lru(conn)
+            conn.commit()
+            return self._load_events(conn, correlation_id)
+
+    def _load_events(self, conn: sqlite3.Connection, correlation_id: str) -> list[dict[str, Any]]:
+        rows = conn.execute(
+            """
+            SELECT event_json FROM live_events
+            WHERE correlation_id = ?
+            ORDER BY seq ASC
+            """,
+            (correlation_id,),
+        ).fetchall()
+        events: list[dict[str, Any]] = []
+        for (raw,) in rows:
+            try:
+                events.append(json.loads(raw))
+            except json.JSONDecodeError:
+                continue
+        return events
+
+    def _evict_lru(self, conn: sqlite3.Connection) -> None:
+        count = conn.execute("SELECT COUNT(*) FROM live_session_meta").fetchone()[0]
+        if count <= _MAX_SESSIONS:
+            return
+        overflow = count - _MAX_SESSIONS
+        rows = conn.execute(
+            """
+            SELECT correlation_id FROM live_session_meta
+            ORDER BY last_touch ASC
+            LIMIT ?
+            """,
+            (overflow,),
+        ).fetchall()
+        for (corr,) in rows:
+            conn.execute("DELETE FROM live_events WHERE correlation_id = ?", (corr,))
+            conn.execute("DELETE FROM live_session_meta WHERE correlation_id = ?", (corr,))
+
+    def is_emitted(self, correlation_id: str, rule_id: str) -> bool:
+        with self._lock:
+            conn = self._get_conn()
+            row = conn.execute(
+                """
+                SELECT 1 FROM live_emitted
+                WHERE correlation_id = ? AND rule_id = ?
+                """,
+                (correlation_id, rule_id),
+            ).fetchone()
+            return row is not None
+
+    def mark_emitted(self, correlation_id: str, rule_id: str, emitted_at: str = "") -> None:
+        with self._lock:
+            conn = self._get_conn()
+            conn.execute(
+                """
+                INSERT OR IGNORE INTO live_emitted (correlation_id, rule_id, emitted_at)
+                VALUES (?, ?, ?)
+                """,
+                (correlation_id, rule_id, emitted_at),
+            )
+            conn.commit()
+
+    def append_host_event(self, host_id: str, event: dict[str, Any]) -> list[dict[str, Any]]:
+        """Append an event to the host-level window and return bounded events."""
+        if not host_id:
+            return []
+        with self._lock:
+            conn = self._get_conn()
+            now = time.time()
+            conn.execute(
+                """
+                INSERT INTO live_host_meta (host_id, last_touch)
+                VALUES (?, ?)
+                ON CONFLICT(host_id) DO UPDATE SET last_touch = excluded.last_touch
+                """,
+                (host_id, now),
+            )
+            row = conn.execute(
+                "SELECT COALESCE(MAX(seq), 0) FROM live_host_events WHERE host_id = ?",
+                (host_id,),
+            ).fetchone()
+            next_seq = int(row[0]) + 1
+            conn.execute(
+                "INSERT INTO live_host_events (host_id, seq, event_json) VALUES (?, ?, ?)",
+                (host_id, next_seq, json.dumps(event, separators=(",", ":"), sort_keys=True)),
+            )
+            min_seq = max(1, next_seq - _MAX_EVENTS_PER_SESSION + 1)
+            conn.execute(
+                "DELETE FROM live_host_events WHERE host_id = ? AND seq < ?",
+                (host_id, min_seq),
+            )
+            conn.commit()
+            return self._load_host_events(conn, host_id)
+
+    def _load_host_events(self, conn: sqlite3.Connection, host_id: str) -> list[dict[str, Any]]:
+        rows = conn.execute(
+            """
+            SELECT event_json FROM live_host_events
+            WHERE host_id = ?
+            ORDER BY seq ASC
+            """,
+            (host_id,),
+        ).fetchall()
+        events: list[dict[str, Any]] = []
+        for (raw,) in rows:
+            try:
+                events.append(json.loads(raw))
+            except json.JSONDecodeError:
+                continue
+        return events
+
+    def is_host_emitted(self, host_id: str, rule_id: str) -> bool:
+        """Has this host-level rule fired recently enough to stay suppressed?
+
+        Host checkpoints expire; session ones do not. A session ends, so a
+        permanent mark is correct there. A host does not: without an expiry the
+        first firing — quite possibly a false positive — silenced the rule on
+        that machine forever, and only deleting a SQLite row brought it back.
+        A recorder that goes permanently blind after one alert is worse than one
+        that occasionally repeats itself.
+        """
+        with self._lock:
+            conn = self._get_conn()
+            row = conn.execute(
+                """
+                SELECT emitted_at FROM live_host_emitted
+                WHERE host_id = ? AND rule_id = ?
+                """,
+                (host_id, rule_id),
+            ).fetchone()
+            if row is None:
+                return False
+            emitted_at = str(row[0] or "")
+            if not emitted_at:
+                # Pre-expiry rows carry no usable timestamp. Treat them as stale
+                # so an upgrade re-arms the rule instead of inheriting silence.
+                conn.execute(
+                    "DELETE FROM live_host_emitted WHERE host_id = ? AND rule_id = ?",
+                    (host_id, rule_id),
+                )
+                conn.commit()
+                return False
+            if _is_expired(emitted_at):
+                conn.execute(
+                    "DELETE FROM live_host_emitted WHERE host_id = ? AND rule_id = ?",
+                    (host_id, rule_id),
+                )
+                conn.commit()
+                return False
+            return True
+
+    def mark_host_emitted(self, host_id: str, rule_id: str, emitted_at: str = "") -> None:
+        """Checkpoint a host-level detection, refreshing the suppression clock.
+
+        Upsert rather than INSERT OR IGNORE: the timestamp drives expiry, so a
+        re-fire must move it forward or the rule would re-arm on the original
+        schedule regardless of recent activity. An absent timestamp falls back to
+        now, so a row always carries a usable clock.
+        """
+        stamp = emitted_at or datetime.now(timezone.utc).isoformat()
+        with self._lock:
+            conn = self._get_conn()
+            conn.execute(
+                """
+                INSERT INTO live_host_emitted (host_id, rule_id, emitted_at)
+                VALUES (?, ?, ?)
+                ON CONFLICT(host_id, rule_id) DO UPDATE SET emitted_at = excluded.emitted_at
+                """,
+                (host_id, rule_id, stamp),
+            )
+            conn.commit()
+
+
+_store: LiveDetectionStore | None = None
+_store_lock = threading.Lock()
+
+
+def get_live_store() -> LiveDetectionStore:
+    global _store
+    if _store is None:
+        with _store_lock:
+            if _store is None:
+                from agentmetry.core.config import settings
+
+                _store = LiveDetectionStore(settings.detection_live_db_path)
+    return _store
+
+
+def reset_live_store_singleton() -> None:
+    """Drop the singleton without deleting data — simulates process restart."""
+    global _store
+    with _store_lock:
+        if _store is not None:
+            _store.close()
+        _store = None

agentmetry/core/audit/detection/models.py

@@ -0,0 +1,53 @@
+"""Data model for correlated detections (sequence rules over a session)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+# Severity ordering for ranking detections most-severe first.
+SEVERITY_RANK: dict[str, int] = {
+    "critical": 0,
+    "high": 1,
+    "medium": 2,
+    "low": 3,
+}
+
+
+@dataclass
+class Detection:
+    """A named, ordered pattern of events — an incident, not a single event.
+
+    `event_ids` are the canonical events that formed the pattern, in order, so
+    the dashboard can link a detection back to the exact rows in the trail.
+    """
+
+    rule_id: str
+    title: str
+    severity: str  # critical | high | medium | low
+    summary: str
+    correlation_id: str
+    tactic_ids: list[str] = field(default_factory=list)
+    technique_ids: list[str] = field(default_factory=list)
+    event_ids: list[str] = field(default_factory=list)
+    first_seen_utc: str = ""
+    last_seen_utc: str = ""
+    # Triage state, attached on read from the disposition store rather than
+    # produced by the rule. A detection nobody dispositioned is an alert, not a
+    # control, so every detection carries the field even when it is untouched.
+    disposition: dict[str, Any] | None = None
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "rule_id": self.rule_id,
+            "title": self.title,
+            "severity": self.severity,
+            "summary": self.summary,
+            "correlation_id": self.correlation_id,
+            "tactic_ids": self.tactic_ids,
+            "technique_ids": self.technique_ids,
+            "event_ids": self.event_ids,
+            "first_seen_utc": self.first_seen_utc,
+            "last_seen_utc": self.last_seen_utc,
+            "disposition": self.disposition,
+        }