cortexdb-mcp 0.2.0__py3-none-any.whl

cortexdb_mcp/insights.py

@@ -0,0 +1,640 @@
+"""Proactive insights engine that analyzes CortexDB episodes to generate actionable intelligence.
+
+Runs heuristic-based analysis over episodes and entities stored in CortexDB to
+surface patterns such as incident spikes, new dependencies, knowledge gaps, and
+deployment risks.  All detection is done with simple temporal and co-occurrence
+analysis -- no LLM calls on the hot path -- so insights can be generated in
+sub-second time.
+
+Typical output examples:
+
+- "payments-service had 3 incidents this week (up from 0 last week)"
+- "New dependency detected: checkout -> stripe-gateway-v2 (since March 10)"
+- "Knowledge gap: nobody documented the billing reconciliation migration"
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import time
+from collections import Counter, defaultdict
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta, timezone
+from enum import Enum
+from typing import Any
+
+import httpx
+
+logger = logging.getLogger("cortexdb_mcp.insights")
+
+
+# ---------------------------------------------------------------------------
+# Data model
+# ---------------------------------------------------------------------------
+
+
+class InsightType(str, Enum):
+    """Categories of proactive insights the engine can produce."""
+
+    incident_spike = "incident_spike"
+    new_dependency = "new_dependency"
+    knowledge_gap = "knowledge_gap"
+    ownership_change = "ownership_change"
+    deployment_risk = "deployment_risk"
+    stale_documentation = "stale_documentation"
+    recurring_issue = "recurring_issue"
+    team_bottleneck = "team_bottleneck"
+
+
+class Severity(str, Enum):
+    """Severity level for an insight."""
+
+    info = "info"
+    warning = "warning"
+    critical = "critical"
+
+
+@dataclass
+class Insight:
+    """A single actionable insight generated by the engine."""
+
+    id: str
+    insight_type: InsightType
+    title: str
+    description: str
+    severity: Severity
+    entities: list[str]
+    evidence: list[str]
+    generated_at: datetime
+    confidence: float
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize the insight to a JSON-safe dictionary."""
+        return {
+            "id": self.id,
+            "insight_type": self.insight_type.value,
+            "title": self.title,
+            "description": self.description,
+            "severity": self.severity.value,
+            "entities": self.entities,
+            "evidence": self.evidence,
+            "generated_at": self.generated_at.isoformat(),
+            "confidence": self.confidence,
+        }
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _make_id(*parts: str) -> str:
+    """Produce a deterministic short insight ID from constituent parts."""
+    raw = ":".join(parts)
+    return "ins_" + hashlib.sha256(raw.encode()).hexdigest()[:12]
+
+
+def _now() -> datetime:
+    """Return the current UTC time."""
+    return datetime.now(timezone.utc)
+
+
+def _week_ago(weeks: int = 1) -> datetime:
+    """Return a datetime ``weeks`` weeks before now."""
+    return _now() - timedelta(weeks=weeks)
+
+
+# ---------------------------------------------------------------------------
+# Engine
+# ---------------------------------------------------------------------------
+
+
+class InsightsEngine:
+    """Analyze CortexDB episodes and generate proactive insights.
+
+    Parameters
+    ----------
+    cortex_url:
+        Base URL of the CortexDB HTTP API.
+    api_key:
+        Optional bearer token for authenticated access.
+    tenant_id:
+        Tenant scope used when querying CortexDB.
+    """
+
+    # Episode types used in heuristics.
+    _INCIDENT_TYPES = {"incident", "alert", "outage", "error", "failure"}
+    _DOCUMENT_TYPES = {"document", "doc", "runbook", "wiki", "documentation", "readme"}
+    _DEPLOY_TYPES = {"deploy", "deployment", "release", "rollout"}
+    _DEPENDENCY_TYPES = {"dependency", "integration", "api_call", "import"}
+
+    def __init__(
+        self,
+        cortex_url: str = "http://localhost:3141",
+        api_key: str | None = None,
+        tenant_id: str | None = None,
+    ) -> None:
+        self.cortex_url = cortex_url.rstrip("/")
+        self.api_key = api_key
+        self.tenant_id = tenant_id
+
+    # -- HTTP helpers -------------------------------------------------------
+
+    def _headers(self) -> dict[str, str]:
+        """Build HTTP headers for CortexDB requests."""
+        headers: dict[str, str] = {"Content-Type": "application/json"}
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+        return headers
+
+    async def _get(self, path: str, params: dict[str, Any] | None = None) -> Any:
+        """Perform a GET request against CortexDB and return parsed JSON."""
+        async with httpx.AsyncClient(
+            base_url=self.cortex_url,
+            headers=self._headers(),
+            timeout=30.0,
+        ) as client:
+            resp = await client.get(path, params=params or {})
+            resp.raise_for_status()
+            return resp.json()
+
+    async def _fetch_episodes(
+        self,
+        *,
+        episode_type: str | None = None,
+        since: datetime | None = None,
+        until: datetime | None = None,
+    ) -> list[dict[str, Any]]:
+        """Fetch episodes from CortexDB with optional type and time filters.
+
+        Returns a list of episode dicts as returned by the ``GET /v1/episodes``
+        endpoint.
+        """
+        params: dict[str, Any] = {}
+        if episode_type is not None:
+            params["episode_type"] = episode_type
+        if since is not None:
+            params["since"] = since.isoformat()
+        if until is not None:
+            params["until"] = until.isoformat()
+        if self.tenant_id is not None:
+            params["tenant_id"] = self.tenant_id
+
+        try:
+            data = await self._get("/v1/episodes", params=params)
+        except (httpx.HTTPStatusError, httpx.RequestError) as exc:
+            logger.warning("Failed to fetch episodes: %s", exc)
+            return []
+
+        if isinstance(data, list):
+            return data
+        return data.get("episodes", data.get("items", []))
+
+    async def _fetch_entities(self) -> list[dict[str, Any]]:
+        """Fetch the entity list from CortexDB."""
+        params: dict[str, Any] = {}
+        if self.tenant_id is not None:
+            params["tenant_id"] = self.tenant_id
+
+        try:
+            data = await self._get("/v1/entities", params=params)
+        except (httpx.HTTPStatusError, httpx.RequestError) as exc:
+            logger.warning("Failed to fetch entities: %s", exc)
+            return []
+
+        if isinstance(data, list):
+            return data
+        return data.get("entities", data.get("items", []))
+
+    # -- Utility extractors -------------------------------------------------
+
+    @staticmethod
+    def _extract_entities(episode: dict[str, Any]) -> list[str]:
+        """Return the list of entity names mentioned in an episode."""
+        entities = episode.get("entities", [])
+        if isinstance(entities, list):
+            return [
+                e.get("name", e) if isinstance(e, dict) else str(e)
+                for e in entities
+            ]
+        return []
+
+    @staticmethod
+    def _episode_type(episode: dict[str, Any]) -> str:
+        """Return the normalised episode type string."""
+        return (episode.get("episode_type") or episode.get("type") or "").lower()
+
+    @staticmethod
+    def _episode_time(episode: dict[str, Any]) -> datetime | None:
+        """Parse the episode timestamp into a datetime, or None on failure."""
+        raw = episode.get("timestamp") or episode.get("created_at") or episode.get("occurred_at")
+        if raw is None:
+            return None
+        if isinstance(raw, datetime):
+            return raw
+        try:
+            return datetime.fromisoformat(str(raw).replace("Z", "+00:00"))
+        except (ValueError, TypeError):
+            return None
+
+    @staticmethod
+    def _episode_id(episode: dict[str, Any]) -> str:
+        """Return a stable identifier for an episode."""
+        return str(
+            episode.get("episode_id")
+            or episode.get("event_id")
+            or episode.get("id")
+            or "unknown"
+        )
+
+    # -- Insight generators -------------------------------------------------
+
+    async def generate_all(self) -> list[Insight]:
+        """Run every insight generator and return the combined results."""
+        results: list[Insight] = []
+        generators = [
+            self.incident_spike_detection,
+            self.new_dependency_detection,
+            self.knowledge_gap_detection,
+            self.deployment_risk_assessment,
+            self.stale_documentation_detection,
+            self.recurring_issue_detection,
+        ]
+        for gen in generators:
+            try:
+                insights = await gen()
+                results.extend(insights)
+            except Exception:
+                logger.exception("Insight generator %s failed", gen.__name__)
+        return results
+
+    # 1. Incident spike -------------------------------------------------------
+
+    async def incident_spike_detection(self) -> list[Insight]:
+        """Compare incident counts this week vs last week per service.
+
+        Generates a warning if a service has more incidents this week than last,
+        and a critical insight if the spike is 3x or more.
+        """
+        now = _now()
+        this_week_start = now - timedelta(weeks=1)
+        last_week_start = now - timedelta(weeks=2)
+
+        all_episodes = await self._fetch_episodes(since=last_week_start)
+
+        this_week: Counter[str] = Counter()
+        last_week: Counter[str] = Counter()
+
+        for ep in all_episodes:
+            if self._episode_type(ep) not in self._INCIDENT_TYPES:
+                continue
+            ts = self._episode_time(ep)
+            if ts is None:
+                continue
+            for entity in self._extract_entities(ep):
+                if ts >= this_week_start:
+                    this_week[entity] += 1
+                elif ts >= last_week_start:
+                    last_week[entity] += 1
+
+        insights: list[Insight] = []
+        all_services = set(this_week) | set(last_week)
+        for svc in sorted(all_services):
+            cur = this_week.get(svc, 0)
+            prev = last_week.get(svc, 0)
+            if cur <= prev:
+                continue
+
+            if prev == 0:
+                severity = Severity.critical if cur >= 3 else Severity.warning
+                desc = (
+                    f"{svc} had {cur} incident(s) this week (up from 0 last week)."
+                )
+            else:
+                ratio = cur / prev
+                severity = Severity.critical if ratio >= 3 else Severity.warning
+                desc = (
+                    f"{svc} had {cur} incident(s) this week "
+                    f"(up from {prev} last week, {ratio:.1f}x increase)."
+                )
+
+            # Collect evidence episode IDs.
+            evidence = [
+                self._episode_id(ep)
+                for ep in all_episodes
+                if self._episode_type(ep) in self._INCIDENT_TYPES
+                and svc in self._extract_entities(ep)
+                and (self._episode_time(ep) or now) >= this_week_start
+            ]
+
+            insights.append(
+                Insight(
+                    id=_make_id("incident_spike", svc, str(cur)),
+                    insight_type=InsightType.incident_spike,
+                    title=f"Incident spike: {svc}",
+                    description=desc,
+                    severity=severity,
+                    entities=[svc],
+                    evidence=evidence,
+                    generated_at=now,
+                    confidence=0.85,
+                )
+            )
+
+        return insights
+
+    # 2. New dependency detection ---------------------------------------------
+
+    async def new_dependency_detection(self) -> list[Insight]:
+        """Find entity pairs that appear together for the first time recently.
+
+        Compares co-occurrences in the last 7 days against co-occurrences in
+        episodes older than 7 days to surface newly-discovered dependencies.
+        """
+        now = _now()
+        recent_cutoff = now - timedelta(days=7)
+        older_cutoff = now - timedelta(days=90)
+
+        all_episodes = await self._fetch_episodes(since=older_cutoff)
+
+        recent_pairs: dict[tuple[str, str], list[str]] = defaultdict(list)
+        older_pairs: set[tuple[str, str]] = set()
+
+        for ep in all_episodes:
+            entities = sorted(set(self._extract_entities(ep)))
+            ts = self._episode_time(ep)
+            if ts is None:
+                continue
+            for i in range(len(entities)):
+                for j in range(i + 1, len(entities)):
+                    pair = (entities[i], entities[j])
+                    if ts >= recent_cutoff:
+                        recent_pairs[pair].append(self._episode_id(ep))
+                    else:
+                        older_pairs.add(pair)
+
+        insights: list[Insight] = []
+        for pair, evidence in sorted(recent_pairs.items()):
+            if pair in older_pairs:
+                continue
+            a, b = pair
+            first_ts = None
+            for ep in all_episodes:
+                ents = self._extract_entities(ep)
+                if a in ents and b in ents:
+                    t = self._episode_time(ep)
+                    if t and (first_ts is None or t < first_ts):
+                        first_ts = t
+
+            since_str = first_ts.strftime("%B %d") if first_ts else "recently"
+            insights.append(
+                Insight(
+                    id=_make_id("new_dep", a, b),
+                    insight_type=InsightType.new_dependency,
+                    title=f"New dependency detected: {a} -> {b}",
+                    description=(
+                        f"New dependency detected: {a} -> {b} (since {since_str}). "
+                        f"These entities were never seen together before the last 7 days."
+                    ),
+                    severity=Severity.info,
+                    entities=[a, b],
+                    evidence=evidence,
+                    generated_at=now,
+                    confidence=0.7,
+                )
+            )
+
+        return insights
+
+    # 3. Knowledge gap detection ----------------------------------------------
+
+    async def knowledge_gap_detection(self) -> list[Insight]:
+        """Find services with many incidents but few document-type episodes.
+
+        A service that is frequently involved in incidents but has little or no
+        documentation is a knowledge gap that should be addressed.
+        """
+        now = _now()
+        since = now - timedelta(days=30)
+        all_episodes = await self._fetch_episodes(since=since)
+
+        incident_counts: Counter[str] = Counter()
+        doc_counts: Counter[str] = Counter()
+
+        for ep in all_episodes:
+            ep_type = self._episode_type(ep)
+            for entity in self._extract_entities(ep):
+                if ep_type in self._INCIDENT_TYPES:
+                    incident_counts[entity] += 1
+                if ep_type in self._DOCUMENT_TYPES:
+                    doc_counts[entity] += 1
+
+        insights: list[Insight] = []
+        for svc, incidents in sorted(incident_counts.items(), key=lambda x: -x[1]):
+            docs = doc_counts.get(svc, 0)
+            if incidents >= 2 and docs == 0:
+                severity = Severity.critical if incidents >= 5 else Severity.warning
+                evidence = [
+                    self._episode_id(ep)
+                    for ep in all_episodes
+                    if self._episode_type(ep) in self._INCIDENT_TYPES
+                    and svc in self._extract_entities(ep)
+                ]
+                insights.append(
+                    Insight(
+                        id=_make_id("knowledge_gap", svc),
+                        insight_type=InsightType.knowledge_gap,
+                        title=f"Knowledge gap: {svc}",
+                        description=(
+                            f"Knowledge gap: nobody documented {svc}. "
+                            f"It had {incidents} incident(s) in the last 30 days "
+                            f"but 0 documentation episodes."
+                        ),
+                        severity=severity,
+                        entities=[svc],
+                        evidence=evidence,
+                        generated_at=now,
+                        confidence=0.75,
+                    )
+                )
+
+        return insights
+
+    # 4. Deployment risk assessment -------------------------------------------
+
+    async def deployment_risk_assessment(self) -> list[Insight]:
+        """Find services with recent deployments and recent incidents.
+
+        A service that was deployed recently and also had incidents is flagged
+        as a deployment risk.
+        """
+        now = _now()
+        since = now - timedelta(days=7)
+        all_episodes = await self._fetch_episodes(since=since)
+
+        deployed: dict[str, list[str]] = defaultdict(list)
+        incident_svcs: dict[str, list[str]] = defaultdict(list)
+
+        for ep in all_episodes:
+            ep_type = self._episode_type(ep)
+            eid = self._episode_id(ep)
+            for entity in self._extract_entities(ep):
+                if ep_type in self._DEPLOY_TYPES:
+                    deployed[entity].append(eid)
+                if ep_type in self._INCIDENT_TYPES:
+                    incident_svcs[entity].append(eid)
+
+        insights: list[Insight] = []
+        for svc in sorted(set(deployed) & set(incident_svcs)):
+            deploy_count = len(deployed[svc])
+            incident_count = len(incident_svcs[svc])
+            severity = Severity.critical if incident_count >= 3 else Severity.warning
+            evidence = deployed[svc] + incident_svcs[svc]
+            insights.append(
+                Insight(
+                    id=_make_id("deploy_risk", svc),
+                    insight_type=InsightType.deployment_risk,
+                    title=f"Deployment risk: {svc}",
+                    description=(
+                        f"{svc} had {deploy_count} deployment(s) and "
+                        f"{incident_count} incident(s) in the last 7 days. "
+                        f"Recent deploys may be correlated with failures."
+                    ),
+                    severity=severity,
+                    entities=[svc],
+                    evidence=evidence,
+                    generated_at=now,
+                    confidence=0.8,
+                )
+            )
+
+        return insights
+
+    # 5. Stale documentation detection ----------------------------------------
+
+    async def stale_documentation_detection(self) -> list[Insight]:
+        """Find document episodes older than 90 days for services that are still active.
+
+        An active service (has any episode in the last 30 days) whose most
+        recent documentation is older than 90 days is flagged.
+        """
+        now = _now()
+        stale_threshold = now - timedelta(days=90)
+        active_since = now - timedelta(days=30)
+
+        # Fetch a wide window to capture old docs.
+        all_episodes = await self._fetch_episodes(since=now - timedelta(days=365))
+
+        latest_doc: dict[str, datetime] = {}
+        active_services: set[str] = set()
+
+        for ep in all_episodes:
+            ep_type = self._episode_type(ep)
+            ts = self._episode_time(ep)
+            if ts is None:
+                continue
+            for entity in self._extract_entities(ep):
+                if ts >= active_since:
+                    active_services.add(entity)
+                if ep_type in self._DOCUMENT_TYPES:
+                    if entity not in latest_doc or ts > latest_doc[entity]:
+                        latest_doc[entity] = ts
+
+        insights: list[Insight] = []
+        for svc in sorted(active_services):
+            last_doc = latest_doc.get(svc)
+            if last_doc is None:
+                continue  # knowledge_gap_detection handles missing docs
+            if last_doc >= stale_threshold:
+                continue
+            days_stale = (now - last_doc).days
+            insights.append(
+                Insight(
+                    id=_make_id("stale_doc", svc),
+                    insight_type=InsightType.stale_documentation,
+                    title=f"Stale documentation: {svc}",
+                    description=(
+                        f"Documentation for {svc} was last updated {days_stale} days ago. "
+                        f"The service is still active -- consider refreshing its docs."
+                    ),
+                    severity=Severity.warning,
+                    entities=[svc],
+                    evidence=[],
+                    generated_at=now,
+                    confidence=0.65,
+                )
+            )
+
+        return insights
+
+    # 6. Recurring issue detection --------------------------------------------
+
+    async def recurring_issue_detection(self) -> list[Insight]:
+        """Find similar incident patterns repeating for the same service.
+
+        If a service has three or more incidents in the last 30 days with
+        overlapping content tokens, the pattern is flagged as recurring.
+        """
+        now = _now()
+        since = now - timedelta(days=30)
+        all_episodes = await self._fetch_episodes(since=since)
+
+        service_incidents: dict[str, list[dict[str, Any]]] = defaultdict(list)
+
+        for ep in all_episodes:
+            if self._episode_type(ep) not in self._INCIDENT_TYPES:
+                continue
+            for entity in self._extract_entities(ep):
+                service_incidents[entity].append(ep)
+
+        insights: list[Insight] = []
+        for svc, incidents in sorted(service_incidents.items()):
+            if len(incidents) < 3:
+                continue
+
+            # Simple token-overlap heuristic: extract content words and find
+            # commonalities across incidents.
+            token_sets: list[set[str]] = []
+            for ep in incidents:
+                content = (ep.get("content") or ep.get("description") or "").lower()
+                tokens = set(content.split())
+                # Remove very short and very common words.
+                tokens = {t for t in tokens if len(t) > 3}
+                token_sets.append(tokens)
+
+            if not token_sets:
+                continue
+
+            # Find tokens that appear in at least half the incidents.
+            all_tokens: Counter[str] = Counter()
+            for ts in token_sets:
+                for t in ts:
+                    all_tokens[t] += 1
+
+            threshold = max(2, len(incidents) // 2)
+            common = [t for t, c in all_tokens.most_common(10) if c >= threshold]
+
+            if not common:
+                continue
+
+            evidence = [self._episode_id(ep) for ep in incidents]
+            pattern_hint = ", ".join(common[:5])
+            insights.append(
+                Insight(
+                    id=_make_id("recurring", svc, pattern_hint),
+                    insight_type=InsightType.recurring_issue,
+                    title=f"Recurring issue: {svc}",
+                    description=(
+                        f"{svc} has {len(incidents)} similar incidents in the last 30 days. "
+                        f"Common keywords: {pattern_hint}."
+                    ),
+                    severity=Severity.warning,
+                    entities=[svc],
+                    evidence=evidence,
+                    generated_at=now,
+                    confidence=0.6,
+                )
+            )
+
+        return insights