agentmetry 0.4.0__py3-none-any.whl

agentmetry/core/bus/outbox.py

@@ -0,0 +1,124 @@
+"""Durable event outbox — SQLite WAL log of every bus event (tokens excluded).
+
+Gives the dashboard replay-on-reconnect (GET /api/v1/events?since=N) and makes
+the event stream a queryable audit source instead of fire-and-forget frames.
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+from pathlib import Path
+from threading import Lock
+from typing import Any
+
+from agentmetry.core.bus.events import Event
+
+_DATA_DIR = Path(__file__).resolve().parents[3] / "data"
+
+
+class EventOutbox:
+    def __init__(self, db_path: str | Path | None = None):
+        path = Path(db_path) if db_path else _DATA_DIR / "events.db"
+        path.parent.mkdir(parents=True, exist_ok=True)
+        self._conn = sqlite3.connect(str(path), check_same_thread=False)
+        self._lock = Lock()
+        with self._lock, self._conn:
+            self._conn.execute("PRAGMA journal_mode=WAL")
+            self._conn.execute(
+                """
+                CREATE TABLE IF NOT EXISTS events (
+                    seq INTEGER PRIMARY KEY,
+                    ts TEXT NOT NULL,
+                    topic TEXT NOT NULL,
+                    session_id TEXT NOT NULL DEFAULT '',
+                    thread_id TEXT NOT NULL DEFAULT '',
+                    payload TEXT NOT NULL
+                )
+                """
+            )
+
+    def append(self, event: Event) -> None:
+        with self._lock, self._conn:
+            self._conn.execute(
+                "INSERT OR REPLACE INTO events (seq, ts, topic, session_id, thread_id, payload) "
+                "VALUES (?, ?, ?, ?, ?, ?)",
+                (
+                    event.seq,
+                    event.ts,
+                    event.topic,
+                    event.session_id,
+                    event.thread_id,
+                    json.dumps(event.payload, default=str),
+                ),
+            )
+
+    def max_seq(self) -> int:
+        with self._lock:
+            row = self._conn.execute("SELECT MAX(seq) FROM events").fetchone()
+        return int(row[0]) if row and row[0] is not None else 0
+
+    def read_since(self, seq: int, limit: int = 500) -> list[dict[str, Any]]:
+        with self._lock:
+            rows = self._conn.execute(
+                "SELECT seq, ts, topic, session_id, thread_id, payload FROM events "
+                "WHERE seq > ? ORDER BY seq LIMIT ?",
+                (seq, limit),
+            ).fetchall()
+        return self._rows_to_dicts(rows)
+
+    def read_between(
+        self,
+        start_ts: str,
+        end_ts: str,
+        *,
+        limit: int = 100_000,
+    ) -> list[dict[str, Any]]:
+        """Return events with ts in [start_ts, end_ts] (ISO-8601 strings, inclusive)."""
+        with self._lock:
+            rows = self._conn.execute(
+                "SELECT seq, ts, topic, session_id, thread_id, payload FROM events "
+                "WHERE ts >= ? AND ts <= ? ORDER BY seq LIMIT ?",
+                (start_ts, end_ts, limit),
+            ).fetchall()
+        return self._rows_to_dicts(rows)
+
+    def read_by_thread_id(self, thread_id: str, *, limit: int = 10_000) -> list[dict[str, Any]]:
+        """Return events for one LangGraph thread / correlation id, ordered by seq."""
+        with self._lock:
+            rows = self._conn.execute(
+                "SELECT seq, ts, topic, session_id, thread_id, payload FROM events "
+                "WHERE thread_id = ? ORDER BY seq LIMIT ?",
+                (thread_id, limit),
+            ).fetchall()
+        return self._rows_to_dicts(rows)
+
+    @staticmethod
+    def _rows_to_dicts(rows) -> list[dict[str, Any]]:
+        out = []
+        for r in rows:
+            try:
+                payload = json.loads(r[5])
+            except json.JSONDecodeError:
+                payload = {}
+            out.append(
+                {
+                    "seq": r[0],
+                    "ts": r[1],
+                    "topic": r[2],
+                    "session_id": r[3],
+                    "thread_id": r[4],
+                    "payload": payload,
+                }
+            )
+        return out
+
+
+_outbox: EventOutbox | None = None
+
+
+def get_outbox() -> EventOutbox:
+    global _outbox
+    if _outbox is None:
+        _outbox = EventOutbox()
+    return _outbox

agentmetry/core/config.py

@@ -0,0 +1,177 @@
+from pathlib import Path
+
+from pydantic import AliasChoices, Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+_ORCHESTRATOR_ROOT = Path(__file__).resolve().parents[2]
+_PACKAGE_ROOT = Path(__file__).resolve().parents[1]
+
+
+def _policy(*parts: str) -> Path:
+    """A policy manifest shipped inside the package.
+
+    These are default configuration, not test data: without them DLP, tool
+    policy and the YAML detection rules are all inert, and a `pip install` used
+    to leave `doctor` opening with three FAILs and secret scanning silently off.
+
+    They live in the package rather than at the repo root because `python -m
+    build` builds the wheel from the sdist, and a force-include reaching outside
+    the project directory does not survive that trip. Config the package needs
+    to run belongs with the package.
+    """
+    return _PACKAGE_ROOT.joinpath("policies", *parts)
+
+
+class Settings(BaseSettings):
+    """Runtime configuration for the Agentmetry SIEM flight recorder."""
+
+    model_config = SettingsConfigDict(
+        env_prefix="AGENTMETRY_",
+        env_file=_ORCHESTRATOR_ROOT / ".env",
+        env_file_encoding="utf-8",
+        extra="ignore",
+    )
+
+    # ------------------------------------------------------------------ SIEM
+    # Operator identity + optional API key protecting ingest/tail/export.
+    operator_id: str = Field(
+        default="",
+        validation_alias=AliasChoices(
+            "OPERATOR_ID",
+            "AGENTMETRY_OPERATOR_ID",
+        ),
+    )
+    fleet_id: str = Field(
+        default="",
+        validation_alias=AliasChoices("AGENTMETRY_FLEET_ID"),
+    )
+    api_key: str = Field(
+        default="",
+        validation_alias=AliasChoices("AGENTMETRY_API_KEY"),
+    )
+
+    # Canonical JSONL trail + query index (system of record for the hook path).
+    audit_export_enabled: bool = Field(
+        default=True,
+        validation_alias=AliasChoices(
+            "AGENTMETRY_AUDIT_EXPORT_ENABLED",
+        ),
+    )
+    audit_export_path: Path = _ORCHESTRATOR_ROOT / "data" / "audit-forward.jsonl"
+    audit_db_path: Path = _ORCHESTRATOR_ROOT / "data" / "audit.db"
+    detection_live_db_path: Path = _ORCHESTRATOR_ROOT / "data" / "detection_live.db"
+    # Triage state. An index over the `detection_disposition` events in the
+    # trail, which remain the record — see core/audit/detection/disposition.py.
+    detection_disposition_db_path: Path = (
+        _ORCHESTRATOR_ROOT / "data" / "detection_disposition.db"
+    )
+    audit_ingest_enabled: bool = True
+    audit_ingest_url: str = "http://127.0.0.1:8000"
+
+    # Forward sinks (all optional; file is the default and never a cloud ledger).
+    audit_sink: str = Field(
+        default="file",
+        validation_alias=AliasChoices("AGENTMETRY_AUDIT_SINK"),
+    )
+    audit_webhook_url: str = Field(
+        default="",
+        validation_alias=AliasChoices(
+            "AGENTMETRY_AUDIT_WEBHOOK_URL",
+        ),
+    )
+    audit_webhook_timeout_seconds: float = 5.0
+    # `canonical` (default) or `cloudevents`. CloudEvents v1.0 structured mode
+    # is what brokers consume; the canonical record still travels whole inside
+    # `data`. Default unchanged so an existing webhook keeps receiving the shape
+    # it was wired up for.
+    audit_webhook_format: str = Field(
+        default="canonical",
+        validation_alias=AliasChoices("AGENTMETRY_AUDIT_WEBHOOK_FORMAT"),
+    )
+    audit_elastic_url: str = Field(
+        default="",
+        validation_alias=AliasChoices(
+            "AGENTMETRY_AUDIT_ELASTIC_URL",
+        ),
+    )
+    audit_elastic_index: str = "logs-agentmetry"
+    audit_elastic_api_key: str = Field(
+        default="",
+        validation_alias=AliasChoices(
+            "ELASTIC_API_KEY",
+            "AGENTMETRY_ELASTIC_API_KEY",
+        ),
+    )
+    audit_elastic_verify_tls: bool = True
+    audit_splunk_hec_url: str = Field(
+        default="",
+        validation_alias=AliasChoices(
+            "AGENTMETRY_AUDIT_SPLUNK_HEC_URL",
+        ),
+    )
+    audit_splunk_hec_token: str = Field(
+        default="",
+        validation_alias=AliasChoices(
+            "SPLUNK_HEC_TOKEN",
+            "AGENTMETRY_SPLUNK_HEC_TOKEN",
+        ),
+    )
+    audit_splunk_index: str = "main"
+    audit_splunk_sourcetype: str = "agentmetry:json"
+    audit_splunk_verify_tls: bool = True
+    audit_alert_webhook_url: str = Field(
+        default="",
+        validation_alias=AliasChoices(
+            "AGENTMETRY_AUDIT_ALERT_WEBHOOK_URL",
+        ),
+    )
+
+    # Semantic DLP — regex scan of tool arguments at the hook boundary.
+    dlp_mode: str = Field(
+        default="log",
+        validation_alias=AliasChoices("AGENTMETRY_DLP_MODE"),
+    )
+    dlp_rules_path: Path = _policy("dlp", "manifest.yaml")
+    dlp_pii: bool = True
+
+    # Tool allow/deny policy — enforced at the hook boundary (like DLP block mode).
+    tool_policy_mode: str = Field(
+        default="log",
+        validation_alias=AliasChoices("AGENTMETRY_TOOL_POLICY_MODE"),
+    )
+    tool_policy_path: Path = (
+        _policy("tool", "manifest.yaml")
+    )
+
+    # Post-ingest policy checks (core/audit/policy.py). Off by default: the
+    # built-in ruleset is a hardcoded starting point, and it annotates only, it
+    # cannot block. Real prevention lives in the hook (DLP block mode).
+    policy_enabled: bool = Field(
+        default=False,
+        validation_alias=AliasChoices("AGENTMETRY_POLICY_ENABLED"),
+    )
+    # off-hours-activity detection. Off by default: "unusual hours" is only a
+    # signal once an operator says which hours are usual, and scheduled jobs
+    # legitimately run at night.
+    detect_off_hours: bool = Field(
+        default=False,
+        validation_alias=AliasChoices("AGENTMETRY_DETECT_OFF_HOURS"),
+    )
+    business_hours: str = Field(
+        default="09-18",  # local start-end, 24h
+        validation_alias=AliasChoices("AGENTMETRY_BUSINESS_HOURS"),
+    )
+    business_tz: str = Field(
+        default="UTC",  # IANA name, e.g. Europe/Athens
+        validation_alias=AliasChoices("AGENTMETRY_BUSINESS_TZ"),
+    )
+
+    detection_rules_path: Path = (
+        _policy("detection", "manifest.yaml")
+    )
+
+    # Demo MCP vault — doctor, drivers.json, vault_fs server (not a skill runtime).
+    vault_path: Path = Path(__file__).resolve().parents[4] / "vault"
+
+
+settings = Settings()