monkeybrain-runtime 1.0.0__py3-none-any.whl

src/introspection/alerting.py

@@ -0,0 +1,142 @@
+"""Alerting — event-driven alert system.
+
+Generate alerts for:
+- Failures
+- Latency thresholds
+- Memory pressure
+- Queue saturation
+- Capability failures
+- Provider failures
+- Database connectivity
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Any, Callable
+from uuid import uuid4
+
+
+class AlertSeverity(str, Enum):
+    INFO = "info"
+    WARNING = "warning"
+    CRITICAL = "critical"
+
+
+class AlertStatus(str, Enum):
+    FIRING = "firing"
+    RESOLVED = "resolved"
+    SILENCED = "silenced"
+
+
+@dataclass
+class Alert:
+    """An alert instance."""
+    
+    alert_id: str = field(default_factory=lambda: f"alert-{uuid4().hex[:8]}")
+    name: str = ""
+    severity: AlertSeverity = AlertSeverity.WARNING
+    status: AlertStatus = AlertStatus.FIRING
+    message: str = ""
+    source: str = ""
+    timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+    metadata: dict[str, Any] = field(default_factory=dict)
+    
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "alert_id": self.alert_id,
+            "name": self.name,
+            "severity": self.severity.value,
+            "status": self.status.value,
+            "message": self.message,
+            "source": self.source,
+            "timestamp": self.timestamp,
+        }
+
+
+class AlertRule:
+    """A rule that triggers alerts."""
+    
+    def __init__(
+        self,
+        name: str,
+        condition: Callable[[dict], bool],
+        severity: AlertSeverity = AlertSeverity.WARNING,
+        message_template: str = "",
+    ):
+        self.name = name
+        self.condition = condition
+        self.severity = severity
+        self.message_template = message_template
+    
+    def evaluate(self, context: dict) -> Alert | None:
+        try:
+            if self.condition(context):
+                return Alert(
+                    name=self.name,
+                    severity=self.severity,
+                    message=self.message_template.format(**context),
+                )
+        except Exception:
+            pass
+        return None
+
+
+class AlertManager:
+    """Alert management system.
+    
+    Responsibilities:
+    - Evaluate alert rules
+    - Fire alerts
+    - Track alert history
+    - Support alert suppression
+    """
+    
+    def __init__(self):
+        self._rules: list[AlertRule] = []
+        self._alerts: list[Alert] = []
+        self._active: dict[str, Alert] = {}
+    
+    def add_rule(self, rule: AlertRule) -> None:
+        self._rules.append(rule)
+    
+    def evaluate(self, context: dict) -> list[Alert]:
+        """Evaluate all rules against context."""
+        new_alerts = []
+        for rule in self._rules:
+            alert = rule.evaluate(context)
+            if alert:
+                self._alerts.append(alert)
+                self._active[alert.alert_id] = alert
+                new_alerts.append(alert)
+        return new_alerts
+    
+    def fire(self, name: str, message: str, severity: AlertSeverity = AlertSeverity.WARNING, **metadata: Any) -> Alert:
+        """Manually fire an alert."""
+        alert = Alert(name=name, severity=severity, message=message, metadata=metadata)
+        self._alerts.append(alert)
+        self._active[alert.alert_id] = alert
+        return alert
+    
+    def resolve(self, alert_id: str) -> bool:
+        """Resolve an active alert."""
+        if alert_id in self._active:
+            self._active[alert_id].status = AlertStatus.RESOLVED
+            del self._active[alert_id]
+            return True
+        return False
+    
+    def get_active(self) -> list[Alert]:
+        return list(self._active.values())
+    
+    def get_history(self, limit: int = 50) -> list[Alert]:
+        return self._alerts[-limit:]
+    
+    def summary(self) -> dict:
+        return {
+            "rules": len(self._rules),
+            "active_alerts": len(self._active),
+            "total_alerts": len(self._alerts),
+        }

src/introspection/health.py

@@ -0,0 +1,101 @@
+"""Health — health monitoring for all subsystems.
+
+Every subsystem reports health:
+- Healthy
+- Degraded
+- Unavailable
+- Unknown
+
+Health checks include dependencies.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any
+from uuid import uuid4
+
+
+@dataclass
+class HealthCheck:
+    """A single health check result."""
+    
+    check_id: str = field(default_factory=lambda: f"health-{uuid4().hex[:8]}")
+    name: str = ""
+    status: str = "healthy"  # healthy | degraded | unhealthy | unknown
+    latency_ms: float = 0.0
+    message: str = ""
+    dependencies: list[dict[str, Any]] = field(default_factory=list)
+    timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+    metadata: dict[str, Any] = field(default_factory=dict)
+    
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "check_id": self.check_id,
+            "name": self.name,
+            "status": self.status,
+            "latency_ms": round(self.latency_ms, 2),
+            "message": self.message,
+            "dependencies": self.dependencies,
+            "timestamp": self.timestamp,
+        }
+
+
+class HealthMonitor:
+    """Health monitoring manager.
+    
+    Responsibilities:
+    - Run health checks
+    - Track health history
+    - Aggregate health status
+    """
+    
+    def __init__(self):
+        self._checks: dict[str, HealthCheck] = {}
+        self._history: list[HealthCheck] = []
+    
+    def check(self, name: str, status: str = "healthy", message: str = "", **metadata: Any) -> HealthCheck:
+        """Record a health check."""
+        check = HealthCheck(
+            name=name,
+            status=status,
+            message=message,
+            metadata=metadata,
+        )
+        self._checks[name] = check
+        self._history.append(check)
+        if len(self._history) > 1000:
+            self._history = self._history[-500:]
+        return check
+    
+    def healthy(self, name: str, **metadata: Any) -> HealthCheck:
+        return self.check(name, "healthy", **metadata)
+    
+    def degraded(self, name: str, message: str = "", **metadata: Any) -> HealthCheck:
+        return self.check(name, "degraded", message, **metadata)
+    
+    def unhealthy(self, name: str, message: str = "", **metadata: Any) -> HealthCheck:
+        return self.check(name, "unhealthy", message, **metadata)
+    
+    def get_health(self, name: str) -> HealthCheck | None:
+        return self._checks.get(name)
+    
+    def overall_status(self) -> str:
+        statuses = [c.status for c in self._checks.values()]
+        if not statuses:
+            return "unknown"
+        if "unhealthy" in statuses:
+            return "unhealthy"
+        if "degraded" in statuses:
+            return "degraded"
+        if all(s == "healthy" for s in statuses):
+            return "healthy"
+        return "unknown"
+    
+    def summary(self) -> dict:
+        return {
+            "overall": self.overall_status(),
+            "checks": {name: check.status for name, check in self._checks.items()},
+            "total_checks": len(self._checks),
+        }

src/introspection/lemon.py

@@ -0,0 +1,243 @@
+"""Lemon — Observability Layer for AgentOS.
+
+Persists all metrics to Elasticsearch for querying and dashboards.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from typing import Any
+
+from src.introspection.tracing import Tracer, Trace, Span
+from src.introspection.metrics import MetricsCollector, Metric
+from src.introspection.logging import StructuredLogger, LogEntry
+from src.introspection.health import HealthMonitor, HealthCheck
+from src.introspection.alerting import AlertManager, Alert, AlertRule, AlertSeverity
+
+
+class Lemon:
+    """Observability manager for AgentOS.
+    
+    Persists all metrics to Elasticsearch for querying.
+    """
+    
+    def __init__(self, elasticsearch_url: str = "http://localhost:9200"):
+        self.tracer = Tracer()
+        self.metrics = MetricsCollector()
+        self.logger = StructuredLogger()
+        self.health = HealthMonitor()
+        self.alerts = AlertManager()
+        self._es_url = elasticsearch_url
+        self._es_client = None
+        self._persist_buffer: list[dict] = []
+    
+    async def connect_elasticsearch(self) -> None:
+        """Connect to Elasticsearch for persistence."""
+        try:
+            from elasticsearch import AsyncElasticsearch
+            self._es_client = AsyncElasticsearch([self._es_url])
+            await self._es_client.ping()
+        except Exception:
+            self._es_client = None
+    
+    async def persist_metrics(self) -> dict[str, Any]:
+        """Persist all metrics to Elasticsearch."""
+        if not self._es_client:
+            return {"status": "no_elasticsearch"}
+        
+        export = self.metrics.export()
+        timestamp = datetime.now(timezone.utc).isoformat()
+        
+        doc = {
+            "timestamp": timestamp,
+            "counters": export.get("counters", {}),
+            "gauges": export.get("gauges", {}),
+            "histograms": export.get("histograms", {}),
+        }
+        
+        try:
+            await self._es_client.index(index="agentos-metrics", document=doc)
+            return {"status": "persisted", "timestamp": timestamp}
+        except Exception as e:
+            return {"status": "error", "error": str(e)}
+    
+    async def persist_traces(self) -> dict[str, Any]:
+        """Persist recent traces to Elasticsearch."""
+        if not self._es_client:
+            return {"status": "no_elasticsearch"}
+        
+        traces = list(self.tracer._traces.values())[-100:]
+        
+        for trace in traces:
+            doc = trace.to_dict()
+            try:
+                await self._es_client.index(index="agentos-traces", document=doc)
+            except Exception:
+                pass
+        
+        return {"status": "persisted", "count": len(traces)}
+    
+    async def persist_logs(self, limit: int = 100) -> dict[str, Any]:
+        """Persist recent logs to Elasticsearch."""
+        if not self._es_client:
+            return {"status": "no_elasticsearch"}
+        
+        logs = self.logger.get_entries(limit=limit)
+        
+        for log in logs:
+            doc = log.to_dict()
+            try:
+                await self._es_client.index(index="agentos-logs", document=doc)
+            except Exception:
+                pass
+        
+        return {"status": "persisted", "count": len(logs)}
+    
+    async def persist_all(self) -> dict[str, Any]:
+        """Persist all observability data to Elasticsearch."""
+        results = {}
+        results["metrics"] = await self.persist_metrics()
+        results["traces"] = await self.persist_traces()
+        results["logs"] = await self.persist_logs()
+        return results
+    
+    async def query_metrics(self, metric_name: str | None = None, size: int = 100) -> list[dict]:
+        """Query metrics from Elasticsearch."""
+        if not self._es_client:
+            return []
+        
+        try:
+            if metric_name:
+                query = {"query": {"term": {"counters." + metric_name: {"exists": True}}}}
+            else:
+                query = {"query": {"match_all": {}}, "sort": [{"timestamp": "desc"}]}
+            
+            result = await self._es_client.search(index="agentos-metrics", body={"size": size, **query})
+            return [hit["_source"] for hit in result["hits"]["hits"]]
+        except Exception:
+            return []
+    
+    async def query_traces(self, trace_id: str | None = None, size: int = 100) -> list[dict]:
+        """Query traces from Elasticsearch."""
+        if not self._es_client:
+            return []
+        
+        try:
+            if trace_id:
+                query = {"query": {"term": {"trace_id": trace_id}}}
+            else:
+                query = {"query": {"match_all": {}}, "sort": [{"timestamp": "desc"}]}
+            
+            result = await self._es_client.search(index="agentos-traces", body={"size": size, **query})
+            return [hit["_source"] for hit in result["hits"]["hits"]]
+        except Exception:
+            return []
+    
+    async def query_logs(self, severity: str | None = None, component: str | None = None, size: int = 100) -> list[dict]:
+        """Query logs from Elasticsearch."""
+        if not self._es_client:
+            return []
+        
+        try:
+            must = []
+            if severity:
+                must.append({"term": {"severity": severity}})
+            if component:
+                must.append({"term": {"component": component}})
+            
+            query = {"query": {"bool": {"must": must or [{"match_all": {}}]}}}
+            result = await self._es_client.search(index="agentos-logs", body={"size": size, "sort": [{"timestamp": "desc"}], **query})
+            return [hit["_source"] for hit in result["hits"]["hits"]]
+        except Exception:
+            return []
+    
+    async def get_dashboard(self) -> dict[str, Any]:
+        """Get dashboard data for all metrics."""
+        metrics = await self.query_metrics(size=1)
+        traces = await self.query_traces(size=10)
+        logs = await self.query_logs(size=10)
+        
+        return {
+            "latest_metrics": metrics[0] if metrics else {},
+            "recent_traces": len(traces),
+            "recent_logs": len(logs),
+            "health": self.health.summary(),
+            "alerts": [a.to_dict() for a in self.alerts.get_active()],
+        }
+    
+    # --- Tracing ---
+    def start_trace(self, name: str = "", **metadata: Any) -> Trace:
+        return self.tracer.start_trace(name, **metadata)
+    
+    def start_span(self, name: str, component: str = "", **attributes: Any) -> Span:
+        return self.tracer.start_span(name, component, **attributes)
+    
+    def finish_span(self, status: str = "ok") -> None:
+        self.tracer.finish_span(status)
+    
+    def finish_trace(self) -> None:
+        self.tracer.finish_trace()
+    
+    # --- Metrics ---
+    def record_metric(self, name: str, value: float, unit: str = "", **tags: str) -> Metric:
+        return self.metrics.record(name, value, unit, **tags)
+    
+    def counter(self, name: str, increment: int = 1, **tags: str) -> None:
+        self.metrics.counter(name, increment, **tags)
+    
+    def gauge(self, name: str, value: float, **tags: str) -> None:
+        self.metrics.gauge(name, value, **tags)
+    
+    def histogram(self, name: str, value: float, **tags: str) -> None:
+        self.metrics.histogram(name, value, **tags)
+    
+    # --- Logging ---
+    def log(self, severity: str, message: str, component: str = "", **kwargs: Any) -> LogEntry:
+        trace = self.tracer.get_current_trace()
+        if trace:
+            kwargs["trace_id"] = trace.trace_id
+        return self.logger.log(severity, message, component=component, **kwargs)
+    
+    def info(self, message: str, component: str = "", **kwargs: Any) -> LogEntry:
+        return self.logger.info(message, component=component, **kwargs)
+    
+    def warn(self, message: str, component: str = "", **kwargs: Any) -> LogEntry:
+        return self.logger.warn(message, component=component, **kwargs)
+    
+    def error(self, message: str, component: str = "", **kwargs: Any) -> LogEntry:
+        return self.logger.error(message, component=component, **kwargs)
+    
+    # --- Health ---
+    def health_check(self, name: str, status: str = "healthy", **metadata: Any) -> HealthCheck:
+        return self.health.check(name, status, **metadata)
+    
+    def overall_health(self) -> str:
+        return self.health.overall_status()
+    
+    # --- Alerts ---
+    def alert(self, name: str, message: str, severity: AlertSeverity = AlertSeverity.WARNING, **metadata: Any) -> Alert:
+        return self.alerts.fire(name, message, severity, **metadata)
+    
+    def add_alert_rule(self, rule: AlertRule) -> None:
+        self.alerts.add_rule(rule)
+    
+    # --- Summary ---
+    def summary(self) -> dict[str, Any]:
+        return {
+            "tracing": self.tracer.summary(),
+            "metrics": self.metrics.summary(),
+            "logging": self.logger.summary(),
+            "health": self.health.summary(),
+            "alerts": self.alerts.summary(),
+            "elasticsearch_connected": self._es_client is not None,
+        }
+    
+    def export(self) -> dict[str, Any]:
+        """Export all observability data."""
+        return {
+            "summary": self.summary(),
+            "metrics": self.metrics.export(),
+            "health": self.health.summary(),
+            "active_alerts": [a.to_dict() for a in self.alerts.get_active()],
+        }

src/introspection/logging.py

@@ -0,0 +1,147 @@
+"""Logging — structured logging for all subsystems.
+
+Every log entry includes:
+- Timestamp
+- Trace ID
+- Span ID
+- Workflow ID
+- Pipeline ID
+- Capability ID
+- User ID
+- Severity
+- Component
+- Duration
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any
+from uuid import uuid4
+
+
+@dataclass
+class LogEntry:
+    """A structured log entry."""
+    
+    log_id: str = field(default_factory=lambda: f"log-{uuid4().hex[:8]}")
+    timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+    severity: str = "info"  # debug | info | warn | error | fatal
+    component: str = ""
+    message: str = ""
+    trace_id: str | None = None
+    span_id: str | None = None
+    workflow_id: str | None = None
+    pipeline_id: str | None = None
+    capability_id: str | None = None
+    user_id: str | None = None
+    duration_ms: float | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    
+    def to_dict(self) -> dict[str, Any]:
+        d = {
+            "log_id": self.log_id,
+            "timestamp": self.timestamp,
+            "severity": self.severity,
+            "component": self.component,
+            "message": self.message,
+        }
+        if self.trace_id: d["trace_id"] = self.trace_id
+        if self.span_id: d["span_id"] = self.span_id
+        if self.pipeline_id: d["pipeline_id"] = self.pipeline_id
+        if self.capability_id: d["capability_id"] = self.capability_id
+        if self.user_id: d["user_id"] = self.user_id
+        if self.duration_ms is not None: d["duration_ms"] = self.duration_ms
+        if self.metadata: d["metadata"] = self.metadata
+        return d
+    
+    def to_json(self) -> str:
+        return json.dumps(self.to_dict(), default=str)
+
+
+class StructuredLogger:
+    """Structured logging manager.
+    
+    Responsibilities:
+    - Create structured log entries
+    - Enrich logs with trace context
+    - Filter by severity
+    - Export logs
+    """
+    
+    def __init__(self, min_level: str = "info"):
+        self._min_level = min_level
+        self._entries: list[LogEntry] = []
+        self._levels = {"debug": 0, "info": 1, "warn": 2, "error": 3, "fatal": 4}
+    
+    def _should_log(self, level: str) -> bool:
+        return self._levels.get(level, 0) >= self._levels.get(self._min_level, 0)
+    
+    def log(
+        self,
+        severity: str,
+        message: str,
+        component: str = "",
+        trace_id: str | None = None,
+        span_id: str | None = None,
+        pipeline_id: str | None = None,
+        capability_id: str | None = None,
+        duration_ms: float | None = None,
+        **metadata: Any,
+    ) -> LogEntry:
+        if not self._should_log(severity):
+            return None
+        
+        entry = LogEntry(
+            severity=severity,
+            component=component,
+            message=message,
+            trace_id=trace_id,
+            span_id=span_id,
+            pipeline_id=pipeline_id,
+            capability_id=capability_id,
+            duration_ms=duration_ms,
+            metadata=metadata,
+        )
+        self._entries.append(entry)
+        return entry
+    
+    def debug(self, message: str, **kwargs: Any) -> LogEntry:
+        return self.log("debug", message, **kwargs)
+    
+    def info(self, message: str, **kwargs: Any) -> LogEntry:
+        return self.log("info", message, **kwargs)
+    
+    def warn(self, message: str, **kwargs: Any) -> LogEntry:
+        return self.log("warn", message, **kwargs)
+    
+    def error(self, message: str, **kwargs: Any) -> LogEntry:
+        return self.log("error", message, **kwargs)
+    
+    def fatal(self, message: str, **kwargs: Any) -> LogEntry:
+        return self.log("fatal", message, **kwargs)
+    
+    def get_entries(
+        self,
+        severity: str | None = None,
+        component: str | None = None,
+        trace_id: str | None = None,
+        limit: int = 100,
+    ) -> list[LogEntry]:
+        entries = self._entries
+        if severity:
+            entries = [e for e in entries if e.severity == severity]
+        if component:
+            entries = [e for e in entries if e.component == component]
+        if trace_id:
+            entries = [e for e in entries if e.trace_id == trace_id]
+        return entries[-limit:]
+    
+    def summary(self) -> dict:
+        counts = {}
+        for e in self._entries:
+            counts[e.severity] = counts.get(e.severity, 0) + 1
+        return {"total": len(self._entries), "by_severity": counts}