cortexdb-connectors 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cortexdb_connectors/__init__.py +40 -0
- cortexdb_connectors/base.py +364 -0
- cortexdb_connectors/cli.py +476 -0
- cortexdb_connectors/confluence/__init__.py +359 -0
- cortexdb_connectors/discord/__init__.py +646 -0
- cortexdb_connectors/github/__init__.py +309 -0
- cortexdb_connectors/gitlab/__init__.py +749 -0
- cortexdb_connectors/google_workspace/__init__.py +1033 -0
- cortexdb_connectors/hubspot/__init__.py +489 -0
- cortexdb_connectors/insights/__init__.py +40 -0
- cortexdb_connectors/insights/api.py +252 -0
- cortexdb_connectors/insights/detectors.py +771 -0
- cortexdb_connectors/insights/engine.py +308 -0
- cortexdb_connectors/intercom/__init__.py +631 -0
- cortexdb_connectors/jira/__init__.py +298 -0
- cortexdb_connectors/linear/__init__.py +705 -0
- cortexdb_connectors/notion/__init__.py +702 -0
- cortexdb_connectors/pagerduty/__init__.py +298 -0
- cortexdb_connectors/salesforce/__init__.py +399 -0
- cortexdb_connectors/servicenow/__init__.py +460 -0
- cortexdb_connectors/slack/__init__.py +331 -0
- cortexdb_connectors/state.py +145 -0
- cortexdb_connectors/teams/__init__.py +656 -0
- cortexdb_connectors/webhooks.py +672 -0
- cortexdb_connectors/worker.py +192 -0
- cortexdb_connectors/zendesk/__init__.py +496 -0
- cortexdb_connectors-0.2.0.dist-info/METADATA +207 -0
- cortexdb_connectors-0.2.0.dist-info/RECORD +31 -0
- cortexdb_connectors-0.2.0.dist-info/WHEEL +5 -0
- cortexdb_connectors-0.2.0.dist-info/entry_points.txt +2 -0
- cortexdb_connectors-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,771 @@
|
|
|
1
|
+
"""Insight detectors for CortexDB proactive analysis.
|
|
2
|
+
|
|
3
|
+
Each detector implements a single analysis pattern, querying the CortexDB
|
|
4
|
+
HTTP API for episodes and entities, then returning zero or more ``Insight``
|
|
5
|
+
objects when a noteworthy pattern is found.
|
|
6
|
+
|
|
7
|
+
Detectors are stateless -- all persistence is handled by the
|
|
8
|
+
``InsightsEngine`` that orchestrates them.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import hashlib
|
|
14
|
+
import logging
|
|
15
|
+
import uuid
|
|
16
|
+
from abc import ABC, abstractmethod
|
|
17
|
+
from collections import Counter, defaultdict
|
|
18
|
+
from datetime import datetime, timedelta, timezone
|
|
19
|
+
from enum import Enum
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
import httpx
|
|
23
|
+
from pydantic import BaseModel, Field
|
|
24
|
+
|
|
25
|
+
logger = logging.getLogger(__name__)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
# ---------------------------------------------------------------------------
|
|
29
|
+
# Data models
|
|
30
|
+
# ---------------------------------------------------------------------------
|
|
31
|
+
|
|
32
|
+
class InsightSeverity(str, Enum):
|
|
33
|
+
"""Severity levels for generated insights."""
|
|
34
|
+
|
|
35
|
+
INFO = "info"
|
|
36
|
+
WARNING = "warning"
|
|
37
|
+
CRITICAL = "critical"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class InsightType(str, Enum):
|
|
41
|
+
"""Canonical insight types produced by the built-in detectors."""
|
|
42
|
+
|
|
43
|
+
INCIDENT_SPIKE = "incident_spike"
|
|
44
|
+
NEW_DEPENDENCY = "new_dependency"
|
|
45
|
+
KNOWLEDGE_GAP = "knowledge_gap"
|
|
46
|
+
STALE_RUNBOOK = "stale_runbook"
|
|
47
|
+
OWNERSHIP_GAP = "ownership_gap"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class Insight(BaseModel):
|
|
51
|
+
"""A single actionable insight generated by a detector."""
|
|
52
|
+
|
|
53
|
+
id: str = Field(default_factory=lambda: str(uuid.uuid4()))
|
|
54
|
+
insight_type: InsightType
|
|
55
|
+
title: str
|
|
56
|
+
description: str
|
|
57
|
+
severity: InsightSeverity
|
|
58
|
+
entities: list[str] = Field(default_factory=list)
|
|
59
|
+
detected_at: datetime = Field(
|
|
60
|
+
default_factory=lambda: datetime.now(timezone.utc)
|
|
61
|
+
)
|
|
62
|
+
confidence: float = Field(ge=0.0, le=1.0, default=1.0)
|
|
63
|
+
dismissed: bool = False
|
|
64
|
+
dismissed_at: datetime | None = None
|
|
65
|
+
metadata: dict[str, Any] = Field(default_factory=dict)
|
|
66
|
+
|
|
67
|
+
def stable_id(self, *parts: str) -> str:
|
|
68
|
+
"""Derive a deterministic id from content so duplicates can be detected.
|
|
69
|
+
|
|
70
|
+
Useful for detectors that want to avoid re-creating the same insight
|
|
71
|
+
on every analysis pass.
|
|
72
|
+
"""
|
|
73
|
+
raw = "|".join(parts)
|
|
74
|
+
return hashlib.sha256(raw.encode()).hexdigest()[:24]
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
# ---------------------------------------------------------------------------
|
|
78
|
+
# CortexDB API client helper
|
|
79
|
+
# ---------------------------------------------------------------------------
|
|
80
|
+
|
|
81
|
+
class CortexAPIClient:
|
|
82
|
+
"""Thin async wrapper around the CortexDB HTTP API used by detectors.
|
|
83
|
+
|
|
84
|
+
Handles authentication headers and pagination.
|
|
85
|
+
"""
|
|
86
|
+
|
|
87
|
+
def __init__(
|
|
88
|
+
self,
|
|
89
|
+
base_url: str,
|
|
90
|
+
api_key: str,
|
|
91
|
+
tenant_id: str = "default",
|
|
92
|
+
timeout: float = 30.0,
|
|
93
|
+
) -> None:
|
|
94
|
+
self.base_url = base_url.rstrip("/")
|
|
95
|
+
self.api_key = api_key
|
|
96
|
+
self.tenant_id = tenant_id
|
|
97
|
+
self.timeout = timeout
|
|
98
|
+
|
|
99
|
+
def _headers(self) -> dict[str, str]:
|
|
100
|
+
return {
|
|
101
|
+
"Authorization": f"Bearer {self.api_key}",
|
|
102
|
+
"X-Tenant-Id": self.tenant_id,
|
|
103
|
+
"Content-Type": "application/json",
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
async def get_episodes(
|
|
107
|
+
self,
|
|
108
|
+
*,
|
|
109
|
+
source: str | None = None,
|
|
110
|
+
episode_type: str | None = None,
|
|
111
|
+
limit: int = 500,
|
|
112
|
+
offset: int = 0,
|
|
113
|
+
) -> list[dict[str, Any]]:
|
|
114
|
+
"""Fetch episodes from ``GET /v1/episodes``.
|
|
115
|
+
|
|
116
|
+
Returns the raw JSON list. Paginates automatically when more
|
|
117
|
+
results are available and *limit* has not been reached.
|
|
118
|
+
"""
|
|
119
|
+
params: dict[str, Any] = {"limit": limit, "offset": offset}
|
|
120
|
+
if source:
|
|
121
|
+
params["source"] = source
|
|
122
|
+
if episode_type:
|
|
123
|
+
params["episode_type"] = episode_type
|
|
124
|
+
|
|
125
|
+
all_episodes: list[dict[str, Any]] = []
|
|
126
|
+
remaining = limit
|
|
127
|
+
|
|
128
|
+
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
|
129
|
+
while remaining > 0:
|
|
130
|
+
params["limit"] = min(remaining, 500)
|
|
131
|
+
resp = await client.get(
|
|
132
|
+
f"{self.base_url}/v1/episodes",
|
|
133
|
+
params=params,
|
|
134
|
+
headers=self._headers(),
|
|
135
|
+
)
|
|
136
|
+
resp.raise_for_status()
|
|
137
|
+
batch = resp.json()
|
|
138
|
+
|
|
139
|
+
# The API may return a list directly or wrap in {"episodes": [...]}.
|
|
140
|
+
items: list[dict[str, Any]]
|
|
141
|
+
if isinstance(batch, list):
|
|
142
|
+
items = batch
|
|
143
|
+
elif isinstance(batch, dict) and "episodes" in batch:
|
|
144
|
+
items = batch["episodes"]
|
|
145
|
+
else:
|
|
146
|
+
items = []
|
|
147
|
+
|
|
148
|
+
if not items:
|
|
149
|
+
break
|
|
150
|
+
|
|
151
|
+
all_episodes.extend(items)
|
|
152
|
+
remaining -= len(items)
|
|
153
|
+
params["offset"] = params.get("offset", 0) + len(items)
|
|
154
|
+
|
|
155
|
+
# If the API returned fewer than requested, we're done.
|
|
156
|
+
if len(items) < params["limit"]:
|
|
157
|
+
break
|
|
158
|
+
|
|
159
|
+
return all_episodes
|
|
160
|
+
|
|
161
|
+
async def get_entities(
|
|
162
|
+
self,
|
|
163
|
+
*,
|
|
164
|
+
limit: int = 1000,
|
|
165
|
+
offset: int = 0,
|
|
166
|
+
) -> list[dict[str, Any]]:
|
|
167
|
+
"""Fetch entities from ``GET /v1/entities``."""
|
|
168
|
+
params: dict[str, Any] = {"limit": limit, "offset": offset}
|
|
169
|
+
|
|
170
|
+
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
|
171
|
+
resp = await client.get(
|
|
172
|
+
f"{self.base_url}/v1/entities",
|
|
173
|
+
params=params,
|
|
174
|
+
headers=self._headers(),
|
|
175
|
+
)
|
|
176
|
+
resp.raise_for_status()
|
|
177
|
+
data = resp.json()
|
|
178
|
+
|
|
179
|
+
if isinstance(data, list):
|
|
180
|
+
return data
|
|
181
|
+
if isinstance(data, dict) and "entities" in data:
|
|
182
|
+
return data["entities"]
|
|
183
|
+
return []
|
|
184
|
+
|
|
185
|
+
async def get_entity_edges(
|
|
186
|
+
self, entity_id: str
|
|
187
|
+
) -> list[dict[str, Any]]:
|
|
188
|
+
"""Fetch edges for a single entity from ``GET /v1/entities/{id}/edges``."""
|
|
189
|
+
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
|
190
|
+
resp = await client.get(
|
|
191
|
+
f"{self.base_url}/v1/entities/{entity_id}/edges",
|
|
192
|
+
headers=self._headers(),
|
|
193
|
+
)
|
|
194
|
+
resp.raise_for_status()
|
|
195
|
+
data = resp.json()
|
|
196
|
+
|
|
197
|
+
if isinstance(data, list):
|
|
198
|
+
return data
|
|
199
|
+
if isinstance(data, dict) and "edges" in data:
|
|
200
|
+
return data["edges"]
|
|
201
|
+
return []
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
# ---------------------------------------------------------------------------
|
|
205
|
+
# Abstract detector
|
|
206
|
+
# ---------------------------------------------------------------------------
|
|
207
|
+
|
|
208
|
+
class BaseDetector(ABC):
|
|
209
|
+
"""Base class for all insight detectors.
|
|
210
|
+
|
|
211
|
+
Subclasses implement ``detect`` which queries CortexDB data via the
|
|
212
|
+
provided ``CortexAPIClient`` and returns a list of ``Insight`` objects.
|
|
213
|
+
"""
|
|
214
|
+
|
|
215
|
+
@abstractmethod
|
|
216
|
+
async def detect(self, client: CortexAPIClient) -> list[Insight]:
|
|
217
|
+
"""Run the detector and return any discovered insights."""
|
|
218
|
+
...
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
# ---------------------------------------------------------------------------
|
|
222
|
+
# Concrete detectors
|
|
223
|
+
# ---------------------------------------------------------------------------
|
|
224
|
+
|
|
225
|
+
class IncidentSpikeDetector(BaseDetector):
|
|
226
|
+
"""Detect unusual increases in incident-type episodes within a time window.
|
|
227
|
+
|
|
228
|
+
Compares the incident count in the recent window to a longer baseline
|
|
229
|
+
window. If the recent rate exceeds the baseline by more than
|
|
230
|
+
``spike_threshold`` standard deviations, a spike insight is emitted.
|
|
231
|
+
|
|
232
|
+
Parameters
|
|
233
|
+
----------
|
|
234
|
+
recent_window_hours:
|
|
235
|
+
Length of the recent window to inspect (default 4 hours).
|
|
236
|
+
baseline_window_hours:
|
|
237
|
+
Length of the historical baseline window (default 168 hours / 7 days).
|
|
238
|
+
spike_threshold:
|
|
239
|
+
Multiplier over the baseline average that constitutes a spike
|
|
240
|
+
(default 2.0x).
|
|
241
|
+
"""
|
|
242
|
+
|
|
243
|
+
def __init__(
|
|
244
|
+
self,
|
|
245
|
+
recent_window_hours: int = 4,
|
|
246
|
+
baseline_window_hours: int = 168,
|
|
247
|
+
spike_threshold: float = 2.0,
|
|
248
|
+
) -> None:
|
|
249
|
+
self.recent_window_hours = recent_window_hours
|
|
250
|
+
self.baseline_window_hours = baseline_window_hours
|
|
251
|
+
self.spike_threshold = spike_threshold
|
|
252
|
+
|
|
253
|
+
async def detect(self, client: CortexAPIClient) -> list[Insight]:
|
|
254
|
+
"""Fetch incident episodes and check for a spike."""
|
|
255
|
+
insights: list[Insight] = []
|
|
256
|
+
now = datetime.now(timezone.utc)
|
|
257
|
+
recent_start = now - timedelta(hours=self.recent_window_hours)
|
|
258
|
+
baseline_start = now - timedelta(hours=self.baseline_window_hours)
|
|
259
|
+
|
|
260
|
+
try:
|
|
261
|
+
episodes = await client.get_episodes(
|
|
262
|
+
episode_type="incident", limit=5000
|
|
263
|
+
)
|
|
264
|
+
except httpx.HTTPError as exc:
|
|
265
|
+
logger.warning("IncidentSpikeDetector: failed to fetch episodes: %s", exc)
|
|
266
|
+
return insights
|
|
267
|
+
|
|
268
|
+
# Bucket episodes by hour.
|
|
269
|
+
baseline_hourly: Counter[int] = Counter()
|
|
270
|
+
recent_count = 0
|
|
271
|
+
|
|
272
|
+
for ep in episodes:
|
|
273
|
+
occurred_at_raw = ep.get("occurred_at")
|
|
274
|
+
if not occurred_at_raw:
|
|
275
|
+
continue
|
|
276
|
+
|
|
277
|
+
try:
|
|
278
|
+
ts = datetime.fromisoformat(occurred_at_raw.replace("Z", "+00:00"))
|
|
279
|
+
except (ValueError, TypeError):
|
|
280
|
+
continue
|
|
281
|
+
|
|
282
|
+
if ts < baseline_start:
|
|
283
|
+
continue
|
|
284
|
+
|
|
285
|
+
hour_key = int((ts - baseline_start).total_seconds() // 3600)
|
|
286
|
+
baseline_hourly[hour_key] += 1
|
|
287
|
+
|
|
288
|
+
if ts >= recent_start:
|
|
289
|
+
recent_count += 1
|
|
290
|
+
|
|
291
|
+
if not baseline_hourly:
|
|
292
|
+
return insights
|
|
293
|
+
|
|
294
|
+
# Compute baseline average hourly rate (excluding the recent window).
|
|
295
|
+
baseline_hours = max(
|
|
296
|
+
self.baseline_window_hours - self.recent_window_hours, 1
|
|
297
|
+
)
|
|
298
|
+
baseline_total = sum(
|
|
299
|
+
count for key, count in baseline_hourly.items()
|
|
300
|
+
if key < (self.baseline_window_hours - self.recent_window_hours)
|
|
301
|
+
)
|
|
302
|
+
avg_hourly = baseline_total / baseline_hours
|
|
303
|
+
recent_rate = recent_count / max(self.recent_window_hours, 1)
|
|
304
|
+
|
|
305
|
+
if avg_hourly == 0 and recent_count > 0:
|
|
306
|
+
# No baseline incidents but we have recent ones -- definite spike.
|
|
307
|
+
severity = InsightSeverity.CRITICAL
|
|
308
|
+
confidence = 0.9
|
|
309
|
+
elif avg_hourly > 0 and recent_rate > avg_hourly * self.spike_threshold:
|
|
310
|
+
ratio = recent_rate / avg_hourly
|
|
311
|
+
if ratio > self.spike_threshold * 2:
|
|
312
|
+
severity = InsightSeverity.CRITICAL
|
|
313
|
+
confidence = min(0.95, 0.7 + (ratio - self.spike_threshold) * 0.05)
|
|
314
|
+
else:
|
|
315
|
+
severity = InsightSeverity.WARNING
|
|
316
|
+
confidence = min(0.9, 0.6 + (ratio - self.spike_threshold) * 0.1)
|
|
317
|
+
else:
|
|
318
|
+
return insights
|
|
319
|
+
|
|
320
|
+
# Collect entity names mentioned in recent incidents.
|
|
321
|
+
entity_names: list[str] = []
|
|
322
|
+
for ep in episodes:
|
|
323
|
+
occurred_raw = ep.get("occurred_at", "")
|
|
324
|
+
try:
|
|
325
|
+
ts = datetime.fromisoformat(occurred_raw.replace("Z", "+00:00"))
|
|
326
|
+
except (ValueError, TypeError):
|
|
327
|
+
continue
|
|
328
|
+
if ts >= recent_start:
|
|
329
|
+
for ent in ep.get("entities", []):
|
|
330
|
+
name = ent.get("display_name") or ent.get("entity_id", "")
|
|
331
|
+
if name and name not in entity_names:
|
|
332
|
+
entity_names.append(name)
|
|
333
|
+
|
|
334
|
+
insight = Insight(
|
|
335
|
+
insight_type=InsightType.INCIDENT_SPIKE,
|
|
336
|
+
title=f"Incident spike: {recent_count} incidents in last {self.recent_window_hours}h",
|
|
337
|
+
description=(
|
|
338
|
+
f"Detected {recent_count} incidents in the last "
|
|
339
|
+
f"{self.recent_window_hours} hours, compared to a baseline "
|
|
340
|
+
f"average of {avg_hourly:.1f}/hour over the past "
|
|
341
|
+
f"{self.baseline_window_hours} hours. "
|
|
342
|
+
f"This is {recent_rate / max(avg_hourly, 0.01):.1f}x the "
|
|
343
|
+
f"normal rate."
|
|
344
|
+
),
|
|
345
|
+
severity=severity,
|
|
346
|
+
entities=entity_names[:20],
|
|
347
|
+
confidence=round(confidence, 2),
|
|
348
|
+
metadata={
|
|
349
|
+
"recent_count": recent_count,
|
|
350
|
+
"baseline_avg_hourly": round(avg_hourly, 3),
|
|
351
|
+
"recent_rate_hourly": round(recent_rate, 3),
|
|
352
|
+
"recent_window_hours": self.recent_window_hours,
|
|
353
|
+
"baseline_window_hours": self.baseline_window_hours,
|
|
354
|
+
},
|
|
355
|
+
)
|
|
356
|
+
insight.id = insight.stable_id("incident_spike", str(now.date()))
|
|
357
|
+
insights.append(insight)
|
|
358
|
+
return insights
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
class NewDependencyDetector(BaseDetector):
|
|
362
|
+
"""Detect newly discovered service dependencies from entity edges.
|
|
363
|
+
|
|
364
|
+
Scans all entities of type ``service`` and inspects their edges.
|
|
365
|
+
Edges with relationship types like ``DependsOn``, ``CallsTo``, or
|
|
366
|
+
``ConsumesFrom`` that were created within the lookback window are
|
|
367
|
+
flagged as new dependencies.
|
|
368
|
+
|
|
369
|
+
Parameters
|
|
370
|
+
----------
|
|
371
|
+
lookback_hours:
|
|
372
|
+
How far back to look for new edges (default 24 hours).
|
|
373
|
+
dependency_edge_types:
|
|
374
|
+
Edge relationship types that represent dependencies.
|
|
375
|
+
"""
|
|
376
|
+
|
|
377
|
+
DEFAULT_DEPENDENCY_EDGES = frozenset({
|
|
378
|
+
"DependsOn", "CallsTo", "ConsumesFrom", "Imports", "Requires",
|
|
379
|
+
})
|
|
380
|
+
|
|
381
|
+
def __init__(
|
|
382
|
+
self,
|
|
383
|
+
lookback_hours: int = 24,
|
|
384
|
+
dependency_edge_types: frozenset[str] | None = None,
|
|
385
|
+
) -> None:
|
|
386
|
+
self.lookback_hours = lookback_hours
|
|
387
|
+
self.dependency_edge_types = (
|
|
388
|
+
dependency_edge_types or self.DEFAULT_DEPENDENCY_EDGES
|
|
389
|
+
)
|
|
390
|
+
|
|
391
|
+
async def detect(self, client: CortexAPIClient) -> list[Insight]:
|
|
392
|
+
"""Scan service entities for new dependency edges."""
|
|
393
|
+
insights: list[Insight] = []
|
|
394
|
+
now = datetime.now(timezone.utc)
|
|
395
|
+
cutoff = now - timedelta(hours=self.lookback_hours)
|
|
396
|
+
|
|
397
|
+
try:
|
|
398
|
+
entities = await client.get_entities(limit=2000)
|
|
399
|
+
except httpx.HTTPError as exc:
|
|
400
|
+
logger.warning("NewDependencyDetector: failed to fetch entities: %s", exc)
|
|
401
|
+
return insights
|
|
402
|
+
|
|
403
|
+
service_entities = [
|
|
404
|
+
e for e in entities
|
|
405
|
+
if (e.get("entity_type") or "").lower() == "service"
|
|
406
|
+
]
|
|
407
|
+
|
|
408
|
+
for entity in service_entities:
|
|
409
|
+
entity_id = entity.get("id") or entity.get("entity_id", "")
|
|
410
|
+
entity_name = (
|
|
411
|
+
entity.get("display_name")
|
|
412
|
+
or entity.get("name")
|
|
413
|
+
or entity_id
|
|
414
|
+
)
|
|
415
|
+
|
|
416
|
+
try:
|
|
417
|
+
edges = await client.get_entity_edges(entity_id)
|
|
418
|
+
except httpx.HTTPError as exc:
|
|
419
|
+
logger.debug(
|
|
420
|
+
"NewDependencyDetector: failed to fetch edges for %s: %s",
|
|
421
|
+
entity_id, exc,
|
|
422
|
+
)
|
|
423
|
+
continue
|
|
424
|
+
|
|
425
|
+
for edge in edges:
|
|
426
|
+
rel_type = edge.get("relationship_type") or edge.get("type", "")
|
|
427
|
+
if rel_type not in self.dependency_edge_types:
|
|
428
|
+
continue
|
|
429
|
+
|
|
430
|
+
created_raw = (
|
|
431
|
+
edge.get("created_at")
|
|
432
|
+
or edge.get("discovered_at")
|
|
433
|
+
or ""
|
|
434
|
+
)
|
|
435
|
+
if not created_raw:
|
|
436
|
+
continue
|
|
437
|
+
|
|
438
|
+
try:
|
|
439
|
+
created_at = datetime.fromisoformat(
|
|
440
|
+
created_raw.replace("Z", "+00:00")
|
|
441
|
+
)
|
|
442
|
+
except (ValueError, TypeError):
|
|
443
|
+
continue
|
|
444
|
+
|
|
445
|
+
if created_at < cutoff:
|
|
446
|
+
continue
|
|
447
|
+
|
|
448
|
+
target_name = (
|
|
449
|
+
edge.get("target_name")
|
|
450
|
+
or edge.get("target_display_name")
|
|
451
|
+
or edge.get("target_id", "unknown")
|
|
452
|
+
)
|
|
453
|
+
|
|
454
|
+
insight = Insight(
|
|
455
|
+
insight_type=InsightType.NEW_DEPENDENCY,
|
|
456
|
+
title=f"New dependency: {entity_name} -> {target_name}",
|
|
457
|
+
description=(
|
|
458
|
+
f"A new '{rel_type}' relationship was discovered "
|
|
459
|
+
f"between '{entity_name}' and '{target_name}' "
|
|
460
|
+
f"at {created_at.isoformat()}. Review this dependency "
|
|
461
|
+
f"to ensure it is expected and documented."
|
|
462
|
+
),
|
|
463
|
+
severity=InsightSeverity.INFO,
|
|
464
|
+
entities=[entity_name, target_name],
|
|
465
|
+
confidence=0.85,
|
|
466
|
+
metadata={
|
|
467
|
+
"source_entity_id": entity_id,
|
|
468
|
+
"target_id": edge.get("target_id", ""),
|
|
469
|
+
"relationship_type": rel_type,
|
|
470
|
+
"created_at": created_raw,
|
|
471
|
+
},
|
|
472
|
+
)
|
|
473
|
+
insight.id = insight.stable_id(
|
|
474
|
+
"new_dep", entity_id, edge.get("target_id", ""), rel_type
|
|
475
|
+
)
|
|
476
|
+
insights.append(insight)
|
|
477
|
+
|
|
478
|
+
return insights
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
class KnowledgeGapDetector(BaseDetector):
|
|
482
|
+
"""Detect entities with high mention count but no documentation episodes.
|
|
483
|
+
|
|
484
|
+
An entity that appears frequently across episodes but has zero
|
|
485
|
+
associated ``document`` episodes likely represents a knowledge gap
|
|
486
|
+
that should be addressed.
|
|
487
|
+
|
|
488
|
+
Parameters
|
|
489
|
+
----------
|
|
490
|
+
min_mention_count:
|
|
491
|
+
Minimum number of episode mentions before an entity is considered
|
|
492
|
+
"high mention" (default 5).
|
|
493
|
+
"""
|
|
494
|
+
|
|
495
|
+
def __init__(self, min_mention_count: int = 5) -> None:
|
|
496
|
+
self.min_mention_count = min_mention_count
|
|
497
|
+
|
|
498
|
+
async def detect(self, client: CortexAPIClient) -> list[Insight]:
|
|
499
|
+
"""Cross-reference entity mention counts against document episodes."""
|
|
500
|
+
insights: list[Insight] = []
|
|
501
|
+
|
|
502
|
+
try:
|
|
503
|
+
all_episodes = await client.get_episodes(limit=5000)
|
|
504
|
+
doc_episodes = await client.get_episodes(
|
|
505
|
+
episode_type="document", limit=5000
|
|
506
|
+
)
|
|
507
|
+
except httpx.HTTPError as exc:
|
|
508
|
+
logger.warning("KnowledgeGapDetector: failed to fetch data: %s", exc)
|
|
509
|
+
return insights
|
|
510
|
+
|
|
511
|
+
# Count mentions per entity across all episodes.
|
|
512
|
+
mention_counts: Counter[str] = Counter()
|
|
513
|
+
entity_display: dict[str, str] = {}
|
|
514
|
+
|
|
515
|
+
for ep in all_episodes:
|
|
516
|
+
for ent in ep.get("entities", []):
|
|
517
|
+
eid = ent.get("entity_id", "")
|
|
518
|
+
if not eid:
|
|
519
|
+
continue
|
|
520
|
+
mention_counts[eid] += 1
|
|
521
|
+
name = ent.get("display_name") or eid
|
|
522
|
+
entity_display[eid] = name
|
|
523
|
+
|
|
524
|
+
# Collect entity ids that appear in documentation episodes.
|
|
525
|
+
documented_entities: set[str] = set()
|
|
526
|
+
for ep in doc_episodes:
|
|
527
|
+
for ent in ep.get("entities", []):
|
|
528
|
+
eid = ent.get("entity_id", "")
|
|
529
|
+
if eid:
|
|
530
|
+
documented_entities.add(eid)
|
|
531
|
+
|
|
532
|
+
# Flag high-mention entities with no documentation.
|
|
533
|
+
for eid, count in mention_counts.most_common():
|
|
534
|
+
if count < self.min_mention_count:
|
|
535
|
+
break
|
|
536
|
+
if eid in documented_entities:
|
|
537
|
+
continue
|
|
538
|
+
|
|
539
|
+
name = entity_display.get(eid, eid)
|
|
540
|
+
severity = (
|
|
541
|
+
InsightSeverity.WARNING if count >= self.min_mention_count * 3
|
|
542
|
+
else InsightSeverity.INFO
|
|
543
|
+
)
|
|
544
|
+
confidence = min(0.95, 0.5 + (count / 100))
|
|
545
|
+
|
|
546
|
+
insight = Insight(
|
|
547
|
+
insight_type=InsightType.KNOWLEDGE_GAP,
|
|
548
|
+
title=f"Knowledge gap: '{name}' has {count} mentions but no docs",
|
|
549
|
+
description=(
|
|
550
|
+
f"Entity '{name}' (id: {eid}) is referenced in {count} "
|
|
551
|
+
f"episodes but has no associated documentation episodes. "
|
|
552
|
+
f"Consider creating documentation to capture institutional "
|
|
553
|
+
f"knowledge about this entity."
|
|
554
|
+
),
|
|
555
|
+
severity=severity,
|
|
556
|
+
entities=[name],
|
|
557
|
+
confidence=round(confidence, 2),
|
|
558
|
+
metadata={
|
|
559
|
+
"entity_id": eid,
|
|
560
|
+
"mention_count": count,
|
|
561
|
+
"min_threshold": self.min_mention_count,
|
|
562
|
+
},
|
|
563
|
+
)
|
|
564
|
+
insight.id = insight.stable_id("knowledge_gap", eid)
|
|
565
|
+
insights.append(insight)
|
|
566
|
+
|
|
567
|
+
return insights
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
class StaleRunbookDetector(BaseDetector):
|
|
571
|
+
"""Detect documents not updated relative to service change frequency.
|
|
572
|
+
|
|
573
|
+
If a service has had recent code changes or incidents but its
|
|
574
|
+
associated documentation episodes are significantly older, the
|
|
575
|
+
documentation is likely stale.
|
|
576
|
+
|
|
577
|
+
Parameters
|
|
578
|
+
----------
|
|
579
|
+
stale_threshold_days:
|
|
580
|
+
Number of days after which a document is considered stale if the
|
|
581
|
+
service has had recent activity (default 30).
|
|
582
|
+
activity_lookback_days:
|
|
583
|
+
How far back to look for recent service activity (default 14).
|
|
584
|
+
"""
|
|
585
|
+
|
|
586
|
+
def __init__(
|
|
587
|
+
self,
|
|
588
|
+
stale_threshold_days: int = 30,
|
|
589
|
+
activity_lookback_days: int = 14,
|
|
590
|
+
) -> None:
|
|
591
|
+
self.stale_threshold_days = stale_threshold_days
|
|
592
|
+
self.activity_lookback_days = activity_lookback_days
|
|
593
|
+
|
|
594
|
+
async def detect(self, client: CortexAPIClient) -> list[Insight]:
|
|
595
|
+
"""Compare document freshness against service activity."""
|
|
596
|
+
insights: list[Insight] = []
|
|
597
|
+
now = datetime.now(timezone.utc)
|
|
598
|
+
stale_cutoff = now - timedelta(days=self.stale_threshold_days)
|
|
599
|
+
activity_cutoff = now - timedelta(days=self.activity_lookback_days)
|
|
600
|
+
|
|
601
|
+
try:
|
|
602
|
+
all_episodes = await client.get_episodes(limit=5000)
|
|
603
|
+
except httpx.HTTPError as exc:
|
|
604
|
+
logger.warning("StaleRunbookDetector: failed to fetch episodes: %s", exc)
|
|
605
|
+
return insights
|
|
606
|
+
|
|
607
|
+
# Group episodes by entity, separating docs from activity.
|
|
608
|
+
entity_docs: dict[str, datetime] = {} # entity_id -> latest doc timestamp
|
|
609
|
+
entity_activity: dict[str, list[datetime]] = defaultdict(list)
|
|
610
|
+
entity_display: dict[str, str] = {}
|
|
611
|
+
activity_types = {"code_change", "incident", "deployment", "alert"}
|
|
612
|
+
|
|
613
|
+
for ep in all_episodes:
|
|
614
|
+
ep_type = ep.get("episode_type", "")
|
|
615
|
+
occurred_raw = ep.get("occurred_at", "")
|
|
616
|
+
try:
|
|
617
|
+
ts = datetime.fromisoformat(occurred_raw.replace("Z", "+00:00"))
|
|
618
|
+
except (ValueError, TypeError):
|
|
619
|
+
continue
|
|
620
|
+
|
|
621
|
+
for ent in ep.get("entities", []):
|
|
622
|
+
eid = ent.get("entity_id", "")
|
|
623
|
+
if not eid:
|
|
624
|
+
continue
|
|
625
|
+
entity_display[eid] = ent.get("display_name") or eid
|
|
626
|
+
|
|
627
|
+
if ep_type == "document":
|
|
628
|
+
existing = entity_docs.get(eid)
|
|
629
|
+
if existing is None or ts > existing:
|
|
630
|
+
entity_docs[eid] = ts
|
|
631
|
+
elif ep_type in activity_types:
|
|
632
|
+
entity_activity[eid].append(ts)
|
|
633
|
+
|
|
634
|
+
# Find entities with recent activity but stale docs.
|
|
635
|
+
for eid, activity_times in entity_activity.items():
|
|
636
|
+
recent_activity = [t for t in activity_times if t >= activity_cutoff]
|
|
637
|
+
if not recent_activity:
|
|
638
|
+
continue
|
|
639
|
+
|
|
640
|
+
latest_doc = entity_docs.get(eid)
|
|
641
|
+
if latest_doc is None:
|
|
642
|
+
# No docs at all -- this is more of a knowledge gap; skip.
|
|
643
|
+
continue
|
|
644
|
+
|
|
645
|
+
if latest_doc >= stale_cutoff:
|
|
646
|
+
# Doc is fresh enough.
|
|
647
|
+
continue
|
|
648
|
+
|
|
649
|
+
name = entity_display.get(eid, eid)
|
|
650
|
+
days_since_doc = (now - latest_doc).days
|
|
651
|
+
recent_count = len(recent_activity)
|
|
652
|
+
|
|
653
|
+
severity = (
|
|
654
|
+
InsightSeverity.WARNING if days_since_doc > self.stale_threshold_days * 2
|
|
655
|
+
else InsightSeverity.INFO
|
|
656
|
+
)
|
|
657
|
+
confidence = min(0.9, 0.5 + recent_count * 0.05 + days_since_doc * 0.002)
|
|
658
|
+
|
|
659
|
+
insight = Insight(
|
|
660
|
+
insight_type=InsightType.STALE_RUNBOOK,
|
|
661
|
+
title=f"Stale runbook: '{name}' docs are {days_since_doc}d old",
|
|
662
|
+
description=(
|
|
663
|
+
f"Entity '{name}' has documentation last updated "
|
|
664
|
+
f"{days_since_doc} days ago, but has had {recent_count} "
|
|
665
|
+
f"activity episodes (code changes, incidents, deployments) "
|
|
666
|
+
f"in the last {self.activity_lookback_days} days. "
|
|
667
|
+
f"The documentation may be outdated."
|
|
668
|
+
),
|
|
669
|
+
severity=severity,
|
|
670
|
+
entities=[name],
|
|
671
|
+
confidence=round(confidence, 2),
|
|
672
|
+
metadata={
|
|
673
|
+
"entity_id": eid,
|
|
674
|
+
"days_since_doc_update": days_since_doc,
|
|
675
|
+
"recent_activity_count": recent_count,
|
|
676
|
+
"latest_doc_date": latest_doc.isoformat(),
|
|
677
|
+
},
|
|
678
|
+
)
|
|
679
|
+
insight.id = insight.stable_id("stale_runbook", eid)
|
|
680
|
+
insights.append(insight)
|
|
681
|
+
|
|
682
|
+
return insights
|
|
683
|
+
|
|
684
|
+
|
|
685
|
+
class OwnershipGapDetector(BaseDetector):
|
|
686
|
+
"""Detect services with no clear owner (no WorksOn/Owns edges).
|
|
687
|
+
|
|
688
|
+
Iterates over service entities and checks for ownership-indicating
|
|
689
|
+
edges. Services without any such edges are flagged.
|
|
690
|
+
|
|
691
|
+
Parameters
|
|
692
|
+
----------
|
|
693
|
+
ownership_edge_types:
|
|
694
|
+
Edge relationship types that indicate ownership.
|
|
695
|
+
"""
|
|
696
|
+
|
|
697
|
+
DEFAULT_OWNERSHIP_EDGES = frozenset({
|
|
698
|
+
"WorksOn", "Owns", "Maintains", "OwnedBy", "MaintainedBy",
|
|
699
|
+
})
|
|
700
|
+
|
|
701
|
+
def __init__(
|
|
702
|
+
self,
|
|
703
|
+
ownership_edge_types: frozenset[str] | None = None,
|
|
704
|
+
) -> None:
|
|
705
|
+
self.ownership_edge_types = (
|
|
706
|
+
ownership_edge_types or self.DEFAULT_OWNERSHIP_EDGES
|
|
707
|
+
)
|
|
708
|
+
|
|
709
|
+
async def detect(self, client: CortexAPIClient) -> list[Insight]:
|
|
710
|
+
"""Check service entities for ownership edges."""
|
|
711
|
+
insights: list[Insight] = []
|
|
712
|
+
|
|
713
|
+
try:
|
|
714
|
+
entities = await client.get_entities(limit=2000)
|
|
715
|
+
except httpx.HTTPError as exc:
|
|
716
|
+
logger.warning("OwnershipGapDetector: failed to fetch entities: %s", exc)
|
|
717
|
+
return insights
|
|
718
|
+
|
|
719
|
+
service_entities = [
|
|
720
|
+
e for e in entities
|
|
721
|
+
if (e.get("entity_type") or "").lower() == "service"
|
|
722
|
+
]
|
|
723
|
+
|
|
724
|
+
for entity in service_entities:
|
|
725
|
+
entity_id = entity.get("id") or entity.get("entity_id", "")
|
|
726
|
+
entity_name = (
|
|
727
|
+
entity.get("display_name")
|
|
728
|
+
or entity.get("name")
|
|
729
|
+
or entity_id
|
|
730
|
+
)
|
|
731
|
+
|
|
732
|
+
try:
|
|
733
|
+
edges = await client.get_entity_edges(entity_id)
|
|
734
|
+
except httpx.HTTPError as exc:
|
|
735
|
+
logger.debug(
|
|
736
|
+
"OwnershipGapDetector: failed to fetch edges for %s: %s",
|
|
737
|
+
entity_id, exc,
|
|
738
|
+
)
|
|
739
|
+
continue
|
|
740
|
+
|
|
741
|
+
has_owner = any(
|
|
742
|
+
(edge.get("relationship_type") or edge.get("type", ""))
|
|
743
|
+
in self.ownership_edge_types
|
|
744
|
+
for edge in edges
|
|
745
|
+
)
|
|
746
|
+
|
|
747
|
+
if has_owner:
|
|
748
|
+
continue
|
|
749
|
+
|
|
750
|
+
insight = Insight(
|
|
751
|
+
insight_type=InsightType.OWNERSHIP_GAP,
|
|
752
|
+
title=f"Ownership gap: '{entity_name}' has no assigned owner",
|
|
753
|
+
description=(
|
|
754
|
+
f"Service '{entity_name}' (id: {entity_id}) has no "
|
|
755
|
+
f"ownership edges ({', '.join(sorted(self.ownership_edge_types))}). "
|
|
756
|
+
f"Unowned services risk slower incident response and "
|
|
757
|
+
f"unclear accountability."
|
|
758
|
+
),
|
|
759
|
+
severity=InsightSeverity.WARNING,
|
|
760
|
+
entities=[entity_name],
|
|
761
|
+
confidence=0.9,
|
|
762
|
+
metadata={
|
|
763
|
+
"entity_id": entity_id,
|
|
764
|
+
"edge_count": len(edges),
|
|
765
|
+
"checked_edge_types": sorted(self.ownership_edge_types),
|
|
766
|
+
},
|
|
767
|
+
)
|
|
768
|
+
insight.id = insight.stable_id("ownership_gap", entity_id)
|
|
769
|
+
insights.append(insight)
|
|
770
|
+
|
|
771
|
+
return insights
|