django-observatory 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- django_observatory/__init__.py +35 -0
- django_observatory/admin.py +69 -0
- django_observatory/alerts.py +212 -0
- django_observatory/apps.py +32 -0
- django_observatory/audit.py +47 -0
- django_observatory/checks.py +82 -0
- django_observatory/conf.py +197 -0
- django_observatory/context.py +156 -0
- django_observatory/correlation/__init__.py +0 -0
- django_observatory/correlation/causal.py +117 -0
- django_observatory/correlation/engine.py +345 -0
- django_observatory/correlation/graph.py +97 -0
- django_observatory/correlation/rules.py +220 -0
- django_observatory/exporters.py +73 -0
- django_observatory/fingerprints.py +74 -0
- django_observatory/health.py +165 -0
- django_observatory/instrumentation/__init__.py +32 -0
- django_observatory/instrumentation/database.py +71 -0
- django_observatory/instrumentation/exceptions.py +75 -0
- django_observatory/instrumentation/http.py +173 -0
- django_observatory/instrumentation/otel.py +115 -0
- django_observatory/internal.py +50 -0
- django_observatory/logging.py +127 -0
- django_observatory/maintenance.py +77 -0
- django_observatory/management/__init__.py +0 -0
- django_observatory/management/commands/__init__.py +0 -0
- django_observatory/management/commands/observability_cleanup.py +21 -0
- django_observatory/management/commands/observability_export.py +33 -0
- django_observatory/management/commands/observability_health.py +19 -0
- django_observatory/management/commands/observability_rebuild_fingerprints.py +37 -0
- django_observatory/management/commands/observability_stats.py +15 -0
- django_observatory/management/commands/observability_test.py +145 -0
- django_observatory/metrics.py +257 -0
- django_observatory/middleware.py +262 -0
- django_observatory/migrations/0001_initial.py +509 -0
- django_observatory/migrations/__init__.py +0 -0
- django_observatory/models.py +562 -0
- django_observatory/observe.py +42 -0
- django_observatory/permissions.py +55 -0
- django_observatory/pipeline.py +176 -0
- django_observatory/redaction.py +119 -0
- django_observatory/routers.py +35 -0
- django_observatory/scheduler.py +77 -0
- django_observatory/search.py +200 -0
- django_observatory/security.py +47 -0
- django_observatory/signals.py +53 -0
- django_observatory/static/django_observatory/app.css +195 -0
- django_observatory/static/django_observatory/app.js +372 -0
- django_observatory/storage/__init__.py +24 -0
- django_observatory/storage/base.py +30 -0
- django_observatory/storage/django.py +175 -0
- django_observatory/system.py +207 -0
- django_observatory/tables.py +182 -0
- django_observatory/templates/django_observatory/_cells.html +1 -0
- django_observatory/templates/django_observatory/_chart.html +1 -0
- django_observatory/templates/django_observatory/_hbars.html +3 -0
- django_observatory/templates/django_observatory/_range.html +4 -0
- django_observatory/templates/django_observatory/_waterfall.html +12 -0
- django_observatory/templates/django_observatory/alerts.html +29 -0
- django_observatory/templates/django_observatory/base.html +55 -0
- django_observatory/templates/django_observatory/dashboard.html +26 -0
- django_observatory/templates/django_observatory/database.html +18 -0
- django_observatory/templates/django_observatory/exception_detail.html +21 -0
- django_observatory/templates/django_observatory/external.html +19 -0
- django_observatory/templates/django_observatory/health.html +12 -0
- django_observatory/templates/django_observatory/incident_detail.html +36 -0
- django_observatory/templates/django_observatory/issue_detail.html +32 -0
- django_observatory/templates/django_observatory/list.html +27 -0
- django_observatory/templates/django_observatory/log_detail.html +26 -0
- django_observatory/templates/django_observatory/metrics.html +15 -0
- django_observatory/templates/django_observatory/performance.html +29 -0
- django_observatory/templates/django_observatory/request_detail.html +46 -0
- django_observatory/templates/django_observatory/search.html +13 -0
- django_observatory/templates/django_observatory/settings.html +17 -0
- django_observatory/templates/django_observatory/system.html +11 -0
- django_observatory/templates/django_observatory/trace_detail.html +17 -0
- django_observatory/templatetags/__init__.py +0 -0
- django_observatory/templatetags/observatory.py +41 -0
- django_observatory/tracing.py +214 -0
- django_observatory/urls.py +33 -0
- django_observatory/views.py +898 -0
- django_observatory-0.1.0.dist-info/METADATA +237 -0
- django_observatory-0.1.0.dist-info/RECORD +86 -0
- django_observatory-0.1.0.dist-info/WHEEL +5 -0
- django_observatory-0.1.0.dist-info/licenses/LICENSE +21 -0
- django_observatory-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""django-observatory: Django-native, offline-first observability.
|
|
2
|
+
|
|
3
|
+
Public API (imported lazily so that importing this package never touches Django
|
|
4
|
+
before the app registry is ready):
|
|
5
|
+
|
|
6
|
+
from django_observatory import observe, span, task, metrics, audit, security
|
|
7
|
+
"""
|
|
8
|
+
import importlib
|
|
9
|
+
|
|
10
|
+
__version__ = "0.1.0"
|
|
11
|
+
|
|
12
|
+
_LAZY = {
|
|
13
|
+
"observe": ("django_observatory.observe", None),
|
|
14
|
+
"metrics": ("django_observatory.metrics", None),
|
|
15
|
+
"audit": ("django_observatory.audit", None),
|
|
16
|
+
"security": ("django_observatory.security", None),
|
|
17
|
+
"span": ("django_observatory.tracing", "span"),
|
|
18
|
+
"task": ("django_observatory.tracing", "task"),
|
|
19
|
+
"bind": ("django_observatory.context", "bind"),
|
|
20
|
+
"capture_exception": ("django_observatory.instrumentation.exceptions", "capture_exception"),
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
__all__ = [*_LAZY, "__version__"]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def __getattr__(name):
|
|
27
|
+
try:
|
|
28
|
+
module, attr = _LAZY[name]
|
|
29
|
+
except KeyError:
|
|
30
|
+
raise AttributeError(f"module 'django_observatory' has no attribute {name!r}") from None
|
|
31
|
+
value = importlib.import_module(module)
|
|
32
|
+
if attr:
|
|
33
|
+
value = getattr(value, attr)
|
|
34
|
+
globals()[name] = value
|
|
35
|
+
return value
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Admin integration. The custom UI is the primary interface; the admin is for
|
|
2
|
+
triage records and for inspecting security/audit data with the usual admin tooling."""
|
|
3
|
+
from django.contrib import admin
|
|
4
|
+
|
|
5
|
+
from .models import Alert, AlertRule, AuditEvent, Incident, Issue, ObservabilityConfiguration, SecurityEvent
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ReadOnlyAdmin(admin.ModelAdmin):
|
|
9
|
+
def has_add_permission(self, request):
|
|
10
|
+
return False
|
|
11
|
+
|
|
12
|
+
def has_change_permission(self, request, obj=None):
|
|
13
|
+
return False
|
|
14
|
+
|
|
15
|
+
def has_delete_permission(self, request, obj=None):
|
|
16
|
+
return False
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@admin.register(Issue)
|
|
20
|
+
class IssueAdmin(admin.ModelAdmin):
|
|
21
|
+
list_display = ("id", "title", "status", "occurrence_count", "first_seen", "last_seen", "assignee")
|
|
22
|
+
list_filter = ("status", "severity")
|
|
23
|
+
search_fields = ("title", "culprit", "exc_type")
|
|
24
|
+
readonly_fields = ("fingerprint", "exc_type", "title", "culprit", "first_seen", "last_seen", "occurrence_count",
|
|
25
|
+
"first_release", "last_release")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@admin.register(AlertRule)
|
|
29
|
+
class AlertRuleAdmin(admin.ModelAdmin):
|
|
30
|
+
list_display = ("name", "metric", "operator", "threshold", "window_minutes", "severity", "enabled")
|
|
31
|
+
list_filter = ("enabled", "severity")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@admin.register(Alert)
|
|
35
|
+
class AlertAdmin(admin.ModelAdmin):
|
|
36
|
+
list_display = ("title", "status", "severity", "value", "started_at", "resolved_at")
|
|
37
|
+
list_filter = ("status", "severity")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@admin.register(Incident)
|
|
41
|
+
class IncidentAdmin(admin.ModelAdmin):
|
|
42
|
+
list_display = ("id", "title", "status", "severity", "confidence", "detected_at", "resolved_at")
|
|
43
|
+
list_filter = ("status", "severity")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@admin.register(SecurityEvent)
|
|
47
|
+
class SecurityEventAdmin(ReadOnlyAdmin):
|
|
48
|
+
list_display = ("timestamp", "event", "severity", "username", "ip_address", "resource")
|
|
49
|
+
list_filter = ("event", "severity")
|
|
50
|
+
search_fields = ("username", "ip_address", "resource")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@admin.register(AuditEvent)
|
|
54
|
+
class AuditEventAdmin(ReadOnlyAdmin):
|
|
55
|
+
"""Read-only by design: audit records cannot be changed or deleted here."""
|
|
56
|
+
|
|
57
|
+
list_display = ("timestamp", "username", "action", "object_type", "object_id", "result")
|
|
58
|
+
list_filter = ("action", "result")
|
|
59
|
+
search_fields = ("username", "object_type", "object_id")
|
|
60
|
+
|
|
61
|
+
def has_view_permission(self, request, obj=None):
|
|
62
|
+
return request.user.is_superuser or request.user.has_perm("django_observatory.view_audit_events")
|
|
63
|
+
|
|
64
|
+
has_module_permission = has_view_permission
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@admin.register(ObservabilityConfiguration)
|
|
68
|
+
class ConfigurationAdmin(admin.ModelAdmin):
|
|
69
|
+
list_display = ("key", "updated_at", "updated_by")
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
"""Alert rules: evaluation, deduplication, notification.
|
|
2
|
+
|
|
3
|
+
One open Alert per rule. While a condition holds the alert is updated, not duplicated;
|
|
4
|
+
external notifications go out when it opens and again only after the rule's cooldown.
|
|
5
|
+
Alerts opened during an active incident are attached to it and notified once, as the
|
|
6
|
+
incident, instead of individually.
|
|
7
|
+
"""
|
|
8
|
+
import datetime
|
|
9
|
+
import json
|
|
10
|
+
import operator
|
|
11
|
+
import urllib.request
|
|
12
|
+
|
|
13
|
+
from django.conf import settings
|
|
14
|
+
from django.core.mail import send_mail
|
|
15
|
+
from django.db.models import Count
|
|
16
|
+
from django.utils import timezone
|
|
17
|
+
|
|
18
|
+
from . import conf, context, internal, metrics
|
|
19
|
+
from .models import Alert, AlertRule, ExceptionRecord, Incident
|
|
20
|
+
from .models import ObservabilityConfiguration as Cfg
|
|
21
|
+
|
|
22
|
+
_OPS = {">": operator.gt, ">=": operator.ge, "<": operator.lt, "<=": operator.le}
|
|
23
|
+
_UNITS = {"error_rate": "%", "p95_latency": " ms", "p99_latency": " ms", "request_rate": "/min",
|
|
24
|
+
"external_failure_rate": "%", "external_p95": " ms", "db_max_ms": " ms",
|
|
25
|
+
"cpu_percent": "%", "memory_percent": "%", "disk_percent": "%"}
|
|
26
|
+
|
|
27
|
+
DEFAULT_RULES = (
|
|
28
|
+
{"name": "High error rate", "metric": "error_rate", "threshold": 5, "severity": "critical"},
|
|
29
|
+
{"name": "Slow responses (P95)", "metric": "p95_latency", "threshold": 2000},
|
|
30
|
+
{"name": "Issue spike", "metric": "issue_occurrences", "threshold": 100, "window_minutes": 60},
|
|
31
|
+
{"name": "External API failures", "metric": "external_failure_rate", "threshold": 10},
|
|
32
|
+
{"name": "Very slow database query", "metric": "db_max_ms", "threshold": 5000},
|
|
33
|
+
{"name": "Disk almost full", "metric": "disk_percent", "threshold": 90, "severity": "critical"},
|
|
34
|
+
{"name": "Server memory pressure", "metric": "memory_percent", "threshold": 90, "window_minutes": 10},
|
|
35
|
+
{"name": "Server CPU saturated", "metric": "cpu_percent", "threshold": 90, "window_minutes": 10},
|
|
36
|
+
)
|
|
37
|
+
_SYSTEM_RULES = ("Disk almost full", "Server memory pressure", "Server CPU saturated")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def ensure_default_rules():
|
|
41
|
+
if not conf.get("ALERTS", "DEFAULT_RULES"):
|
|
42
|
+
return
|
|
43
|
+
# two flags, so installations created before server monitoring existed gain its rules once
|
|
44
|
+
for key, wanted in (("defaults:alert_rules", lambda n: n not in _SYSTEM_RULES),
|
|
45
|
+
("defaults:alert_rules:system", lambda n: n in _SYSTEM_RULES)):
|
|
46
|
+
_, created = Cfg.objects.get_or_create(key=key)
|
|
47
|
+
if created:
|
|
48
|
+
for spec in DEFAULT_RULES:
|
|
49
|
+
if wanted(spec["name"]):
|
|
50
|
+
AlertRule.objects.get_or_create(name=spec["name"],
|
|
51
|
+
defaults={k: v for k, v in spec.items() if k != "name"})
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# -- measurement ----------------------------------------------------------------
|
|
55
|
+
def measure(rule, now=None):
|
|
56
|
+
"""(value, detail) for the rule's window, or None when there is too little data to judge."""
|
|
57
|
+
now = now or timezone.now()
|
|
58
|
+
start = now - datetime.timedelta(minutes=rule.window_minutes)
|
|
59
|
+
m = rule.metric
|
|
60
|
+
if m == "error_rate":
|
|
61
|
+
by = metrics.by_label("http.requests", "status", start, now)
|
|
62
|
+
total = sum(a.sum for a in by.values())
|
|
63
|
+
if total < 10:
|
|
64
|
+
return None
|
|
65
|
+
errors = by["5xx"].sum if "5xx" in by else 0
|
|
66
|
+
return errors / total * 100, f"{errors:.0f} of {total:.0f} requests failed"
|
|
67
|
+
if m in ("p95_latency", "p99_latency"):
|
|
68
|
+
agg = metrics.total("http.duration", start, now)
|
|
69
|
+
if agg.count < 10:
|
|
70
|
+
return None
|
|
71
|
+
return agg.quantile(0.95 if m == "p95_latency" else 0.99), f"over {agg.count} requests"
|
|
72
|
+
if m == "request_rate":
|
|
73
|
+
return metrics.total("http.requests", start, now).sum / rule.window_minutes, ""
|
|
74
|
+
if m == "exception_count":
|
|
75
|
+
return metrics.total("exceptions", start, now).sum, ""
|
|
76
|
+
if m == "issue_occurrences":
|
|
77
|
+
top = (ExceptionRecord.objects.filter(timestamp__gte=start).values("issue_id", "issue__title")
|
|
78
|
+
.annotate(n=Count("id")).order_by("-n").first())
|
|
79
|
+
return (top["n"], top["issue__title"]) if top else (0, "")
|
|
80
|
+
if m in ("external_failure_rate", "external_p95"):
|
|
81
|
+
worst = None
|
|
82
|
+
if m == "external_p95":
|
|
83
|
+
for service, agg in metrics.by_label("ext.duration", "service", start, now).items():
|
|
84
|
+
if agg.count >= 5 and (worst is None or agg.quantile(0.95) > worst[0]):
|
|
85
|
+
worst = (agg.quantile(0.95), f"service {service}")
|
|
86
|
+
else:
|
|
87
|
+
calls = metrics._scan("ext.calls", start, now, lambda n, t, lb: (lb.get("service"), lb.get("outcome")))
|
|
88
|
+
for service in {s for s, _ in calls}:
|
|
89
|
+
ok = calls.get((service, "ok"))
|
|
90
|
+
err = calls.get((service, "error"))
|
|
91
|
+
total = (ok.sum if ok else 0) + (err.sum if err else 0)
|
|
92
|
+
rate = (err.sum if err else 0) / total * 100 if total else 0
|
|
93
|
+
if total >= 5 and (worst is None or rate > worst[0]):
|
|
94
|
+
worst = (rate, f"service {service}")
|
|
95
|
+
return worst
|
|
96
|
+
if m == "slow_query_count":
|
|
97
|
+
return metrics.total("db.slow_queries", start, now).sum, ""
|
|
98
|
+
if m == "db_max_ms":
|
|
99
|
+
agg = metrics.total("db.duration", start, now)
|
|
100
|
+
return (agg.max, "") if agg.count else None
|
|
101
|
+
if m == "login_failures":
|
|
102
|
+
return metrics.total("security.events", start, now, event="LOGIN_FAILURE").sum, ""
|
|
103
|
+
if m in ("cpu_percent", "memory_percent", "disk_percent"):
|
|
104
|
+
from . import system
|
|
105
|
+
|
|
106
|
+
w = system.worst(start, now + datetime.timedelta(minutes=1))
|
|
107
|
+
if m == "disk_percent":
|
|
108
|
+
return (w["disk"], f"volume {w['disk_mount']} on {w['disk_host']}") if w["disk"] is not None else None
|
|
109
|
+
key = "cpu" if m == "cpu_percent" else "memory"
|
|
110
|
+
return (w[key], f"host {w[key + '_host']}") if w[key] is not None else None
|
|
111
|
+
if m == "custom" and rule.custom_metric:
|
|
112
|
+
return metrics.total(rule.custom_metric, start, now).sum, rule.custom_metric
|
|
113
|
+
return None
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
# -- notification ---------------------------------------------------------------
|
|
117
|
+
def _email(subject, body, payload):
|
|
118
|
+
to = conf.get("ALERTS", "EMAIL_TO")
|
|
119
|
+
if to:
|
|
120
|
+
send_mail(subject, body, getattr(settings, "DEFAULT_FROM_EMAIL", None), list(to), fail_silently=False)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _webhook(subject, body, payload):
|
|
124
|
+
url = conf.get("ALERTS", "WEBHOOK_URL")
|
|
125
|
+
if url:
|
|
126
|
+
if not (isinstance(url, str) and url.startswith(("http://", "https://"))):
|
|
127
|
+
internal.warn("alerts.webhook", f"invalid webhook scheme: {url}")
|
|
128
|
+
return
|
|
129
|
+
data = json.dumps({"text": f"{subject}\n{body}", **payload}, default=str).encode()
|
|
130
|
+
req = urllib.request.Request(url, data=data, headers={"Content-Type": "application/json"}, method="POST")
|
|
131
|
+
urllib.request.urlopen(req, timeout=5).close() # noqa: S310 - operator-configured URL
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
# Add channels here ("teams", "slack", "sms"): fn(subject, body, payload).
|
|
135
|
+
CHANNELS = {"email": _email, "webhook": _webhook}
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def notify(subject, body, payload, channels=None):
|
|
139
|
+
"""Best effort on every channel; one failing channel never blocks another."""
|
|
140
|
+
if channels is None:
|
|
141
|
+
channels = [c for c, on in (("email", conf.get("ALERTS", "EMAIL_TO")),
|
|
142
|
+
("webhook", conf.get("ALERTS", "WEBHOOK_URL"))) if on]
|
|
143
|
+
sent = []
|
|
144
|
+
with context.suppressed():
|
|
145
|
+
for name in channels:
|
|
146
|
+
fn = CHANNELS.get(name)
|
|
147
|
+
if fn is None:
|
|
148
|
+
continue
|
|
149
|
+
try:
|
|
150
|
+
fn(subject, body, payload)
|
|
151
|
+
sent.append(name)
|
|
152
|
+
except Exception as exc: # noqa: BLE001
|
|
153
|
+
internal.warn(f"alerts.{name}", "notification failed", exc)
|
|
154
|
+
return sent
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def notify_incident(incident):
|
|
158
|
+
body = (f"Probable cause ({incident.confidence} confidence, inferred): {incident.probable_cause}\n"
|
|
159
|
+
f"Impact: {incident.affected_requests} requests, {incident.affected_users} users, "
|
|
160
|
+
f"{incident.error_count} errors\nRecommended: {incident.recommendation}")
|
|
161
|
+
notify(f"[{conf.get('SERVICE_NAME')}] Incident #{incident.pk}: {incident.title}", body,
|
|
162
|
+
{"type": "incident", "id": incident.pk, "severity": incident.severity, "status": incident.status})
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
# -- evaluation -----------------------------------------------------------------
|
|
166
|
+
def evaluate(now=None):
|
|
167
|
+
"""Evaluate every enabled rule once. Returns the alerts opened in this run."""
|
|
168
|
+
now = now or timezone.now()
|
|
169
|
+
opened = []
|
|
170
|
+
with context.suppressed():
|
|
171
|
+
ensure_default_rules()
|
|
172
|
+
incident = Incident.objects.exclude(status=Incident.Status.RESOLVED).order_by("-detected_at").first()
|
|
173
|
+
for rule in AlertRule.objects.filter(enabled=True):
|
|
174
|
+
try:
|
|
175
|
+
result = measure(rule, now)
|
|
176
|
+
except Exception as exc: # noqa: BLE001
|
|
177
|
+
internal.warn("alerts.measure", f"rule '{rule.name}' failed", exc)
|
|
178
|
+
continue
|
|
179
|
+
current = (Alert.objects.filter(rule=rule).exclude(status=Alert.Status.RESOLVED)
|
|
180
|
+
.order_by("-started_at").first())
|
|
181
|
+
firing = result is not None and result[0] is not None and _OPS[rule.operator](result[0], rule.threshold)
|
|
182
|
+
if not firing:
|
|
183
|
+
if current and result is not None: # "no data" never auto-resolves
|
|
184
|
+
current.status, current.resolved_at = Alert.Status.RESOLVED, now
|
|
185
|
+
current.save(update_fields=["status", "resolved_at"])
|
|
186
|
+
continue
|
|
187
|
+
value, detail = result
|
|
188
|
+
unit = _UNITS.get(rule.metric, "")
|
|
189
|
+
message = (f"{rule.get_metric_display()} is {value:.1f}{unit} "
|
|
190
|
+
f"({rule.operator} {rule.threshold:g}{unit}) over {rule.window_minutes} min"
|
|
191
|
+
+ (f": {detail}" if detail else ""))
|
|
192
|
+
if current:
|
|
193
|
+
current.value, current.last_seen, current.message = value, now, message
|
|
194
|
+
current.evaluations += 1
|
|
195
|
+
due = (current.status == Alert.Status.FIRING and current.incident_id is None and current.notified_at
|
|
196
|
+
and now - current.notified_at >= datetime.timedelta(minutes=rule.cooldown_minutes))
|
|
197
|
+
if due:
|
|
198
|
+
notify(f"[{conf.get('SERVICE_NAME')}] STILL FIRING: {rule.name}", message,
|
|
199
|
+
{"type": "alert", "id": current.pk, "severity": rule.severity}, rule.channels or None)
|
|
200
|
+
current.notified_at = now
|
|
201
|
+
current.save()
|
|
202
|
+
continue
|
|
203
|
+
alert = Alert.objects.create(rule=rule, incident=incident, severity=rule.severity, title=rule.name,
|
|
204
|
+
message=message, value=value, threshold=rule.threshold,
|
|
205
|
+
started_at=now, last_seen=now)
|
|
206
|
+
opened.append(alert)
|
|
207
|
+
if incident is None: # grouped under the incident otherwise: one notification, not many
|
|
208
|
+
notify(f"[{conf.get('SERVICE_NAME')}] ALERT: {rule.name}", message,
|
|
209
|
+
{"type": "alert", "id": alert.pk, "severity": rule.severity}, rule.channels or None)
|
|
210
|
+
alert.notified_at = now
|
|
211
|
+
alert.save(update_fields=["notified_at"])
|
|
212
|
+
return opened
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from django.apps import AppConfig
|
|
2
|
+
from django.core.signals import setting_changed
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class ObservabilityConfig(AppConfig):
|
|
6
|
+
name = "django_observatory"
|
|
7
|
+
verbose_name = "Observability"
|
|
8
|
+
default_auto_field = "django.db.models.BigAutoField"
|
|
9
|
+
|
|
10
|
+
def ready(self):
|
|
11
|
+
from . import checks, conf, instrumentation, internal, metrics, pipeline, redaction, signals, storage # noqa: F401 (checks: registers system checks)
|
|
12
|
+
from . import logging as obs_logging
|
|
13
|
+
|
|
14
|
+
def _reset(setting=None, **_):
|
|
15
|
+
if setting is None or setting.startswith("OBSERVABILITY"):
|
|
16
|
+
conf.reset()
|
|
17
|
+
redaction.reset()
|
|
18
|
+
storage.reset()
|
|
19
|
+
pipeline.reset()
|
|
20
|
+
metrics.reset()
|
|
21
|
+
|
|
22
|
+
setting_changed.connect(_reset, dispatch_uid="django_observatory.settings", weak=False)
|
|
23
|
+
|
|
24
|
+
if not conf.get("ENABLED"):
|
|
25
|
+
return
|
|
26
|
+
try:
|
|
27
|
+
instrumentation.install()
|
|
28
|
+
signals.connect()
|
|
29
|
+
if conf.get("LOGGING", "AUTO_ATTACH"):
|
|
30
|
+
obs_logging.auto_attach()
|
|
31
|
+
except Exception as exc: # noqa: BLE001 - never prevent the project from starting
|
|
32
|
+
internal.warn("startup", "observability could not initialise", exc)
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Audit trail API. Records are append-only and hash-chained (see models.AuditEvent).
|
|
2
|
+
|
|
3
|
+
from django_observatory import audit
|
|
4
|
+
audit.record(action="UPDATE", object_type="Material", object_id="A1023",
|
|
5
|
+
changes={"quantity": {"before": 120, "after": 80}})
|
|
6
|
+
"""
|
|
7
|
+
import random
|
|
8
|
+
import time
|
|
9
|
+
|
|
10
|
+
from . import conf, context, internal, metrics, pipeline, redaction
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@internal.safe
|
|
14
|
+
def record(action, object_type="", object_id="", *, changes=None, before=None, after=None,
|
|
15
|
+
result="SUCCESS", reason="", user=None, username="", **metadata):
|
|
16
|
+
if not conf.enabled("AUDIT") or context.is_suppressed():
|
|
17
|
+
return
|
|
18
|
+
rate = conf.get("SAMPLING", "audit", default=1.0)
|
|
19
|
+
if rate < 1.0 and random.random() >= rate:
|
|
20
|
+
return
|
|
21
|
+
ctx = context.current()
|
|
22
|
+
rid, tid, sid = context.ids()
|
|
23
|
+
user_id = ctx.user_id if ctx else ""
|
|
24
|
+
uname = username or (ctx.username if ctx else "")
|
|
25
|
+
if user is not None and getattr(user, "pk", None) is not None:
|
|
26
|
+
user_id, uname = str(user.pk), uname or user.get_username()
|
|
27
|
+
if changes and before is None and after is None:
|
|
28
|
+
# derive before/after from {"field": {"before": x, "after": y}}
|
|
29
|
+
pairs = {k: v for k, v in changes.items() if isinstance(v, dict) and {"before", "after"} <= set(v)}
|
|
30
|
+
if pairs:
|
|
31
|
+
before = {k: v["before"] for k, v in pairs.items()}
|
|
32
|
+
after = {k: v["after"] for k, v in pairs.items()}
|
|
33
|
+
action = str(action).upper()
|
|
34
|
+
metrics.increment("audit.events", labels={"action": action})
|
|
35
|
+
if ctx is not None:
|
|
36
|
+
ctx.crumb("audit", f"{action} {object_type} #{object_id}")
|
|
37
|
+
clean = redaction.clean
|
|
38
|
+
pipeline.enqueue("audit", {
|
|
39
|
+
"timestamp": context.to_dt(time.time()), "user_id": user_id, "username": uname, "action": action,
|
|
40
|
+
"object_type": str(object_type), "object_id": str(object_id),
|
|
41
|
+
"before": clean(before) if before is not None else None,
|
|
42
|
+
"after": clean(after) if after is not None else None,
|
|
43
|
+
"changes": clean(changes) if changes is not None else None,
|
|
44
|
+
"ip_address": ctx.ip if ctx else "", "result": str(result).upper(),
|
|
45
|
+
"reason": redaction.text(reason, 500), "metadata": clean(metadata),
|
|
46
|
+
"request_id": rid, "trace_id": tid, "span_id": sid,
|
|
47
|
+
}, high=True)
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""``manage.py check`` validation of the observability configuration."""
|
|
2
|
+
import re
|
|
3
|
+
|
|
4
|
+
from django.conf import settings
|
|
5
|
+
from django.core.checks import Error, Warning, register
|
|
6
|
+
from django.utils.module_loading import import_string
|
|
7
|
+
|
|
8
|
+
from . import conf
|
|
9
|
+
from .storage import BUILTIN
|
|
10
|
+
|
|
11
|
+
MW = "django_observatory.middleware.ObservabilityMiddleware"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@register()
|
|
15
|
+
def check_configuration(app_configs, **kwargs):
|
|
16
|
+
out = []
|
|
17
|
+
try:
|
|
18
|
+
conf.reset()
|
|
19
|
+
cfg = conf.all()
|
|
20
|
+
except Exception as exc: # noqa: BLE001
|
|
21
|
+
return [Error(f"OBSERVABILITY settings could not be loaded: {exc}", id="observability.E001")]
|
|
22
|
+
|
|
23
|
+
for key, rate in cfg["SAMPLING"].items():
|
|
24
|
+
if not isinstance(rate, (int, float)) or isinstance(rate, bool) or not 0 <= rate <= 1:
|
|
25
|
+
out.append(Error(f"SAMPLING['{key}'] must be a number between 0 and 1 (got {rate!r}).",
|
|
26
|
+
id="observability.E002"))
|
|
27
|
+
for key in ("security", "audit"):
|
|
28
|
+
if isinstance(cfg["SAMPLING"].get(key), (int, float)) and cfg["SAMPLING"][key] < 1:
|
|
29
|
+
out.append(Warning(f"SAMPLING['{key}'] is below 1.0: {key} events will be lost.",
|
|
30
|
+
id="observability.W001"))
|
|
31
|
+
for key, days in cfg["RETENTION"].items():
|
|
32
|
+
if not isinstance(days, int) or isinstance(days, bool) or days < 1:
|
|
33
|
+
out.append(Error(f"RETENTION['{key}'] must be a positive number of days (got {days!r}).",
|
|
34
|
+
id="observability.E003"))
|
|
35
|
+
|
|
36
|
+
backend = cfg["STORAGE"]["BACKEND"]
|
|
37
|
+
try:
|
|
38
|
+
import_string(BUILTIN.get(backend, backend))
|
|
39
|
+
except Exception: # noqa: BLE001
|
|
40
|
+
out.append(Error(f"STORAGE['BACKEND'] '{backend}' is not a known backend or importable class.",
|
|
41
|
+
id="observability.E004"))
|
|
42
|
+
alias = cfg["STORAGE"]["DATABASE_ALIAS"] or "default"
|
|
43
|
+
if alias not in settings.DATABASES:
|
|
44
|
+
out.append(Error(f"STORAGE['DATABASE_ALIAS'] '{alias}' is not defined in DATABASES.", id="observability.E005"))
|
|
45
|
+
elif alias != "default" and not any("ObservabilityRouter" in r for r in map(str, settings.DATABASE_ROUTERS)):
|
|
46
|
+
out.append(Warning(
|
|
47
|
+
"STORAGE['DATABASE_ALIAS'] is set but django_observatory.routers.ObservabilityRouter is not in "
|
|
48
|
+
"DATABASE_ROUTERS; telemetry will still be written to 'default'.", id="observability.W002"))
|
|
49
|
+
|
|
50
|
+
count = list(settings.MIDDLEWARE).count(MW)
|
|
51
|
+
if count == 0 and cfg["ENABLED"]:
|
|
52
|
+
out.append(Warning(f"{MW} is not in MIDDLEWARE: requests, traces and SQL correlation are disabled.",
|
|
53
|
+
id="observability.W003"))
|
|
54
|
+
elif count > 1:
|
|
55
|
+
out.append(Error(f"{MW} is listed {count} times in MIDDLEWARE.", id="observability.E006"))
|
|
56
|
+
|
|
57
|
+
if cfg["PERMISSION_MODE"] not in ("staff", "permissions"):
|
|
58
|
+
out.append(Error("PERMISSION_MODE must be 'staff' or 'permissions'.", id="observability.E007"))
|
|
59
|
+
for pattern in cfg["REDACTION"].get("patterns", ()):
|
|
60
|
+
try:
|
|
61
|
+
re.compile(pattern)
|
|
62
|
+
except re.error as exc:
|
|
63
|
+
out.append(Error(f"REDACTION pattern {pattern!r} is not a valid regular expression: {exc}",
|
|
64
|
+
id="observability.E008"))
|
|
65
|
+
|
|
66
|
+
insecure = [name for name, on in (
|
|
67
|
+
("REQUESTS.CAPTURE_BODY", cfg["REQUESTS"]["CAPTURE_BODY"]),
|
|
68
|
+
("DATABASE.CAPTURE_PARAMS", cfg["DATABASE"]["CAPTURE_PARAMS"]),
|
|
69
|
+
("REQUESTS.CAPTURE_QUERY_PARAMS", cfg["REQUESTS"]["CAPTURE_QUERY_PARAMS"]),
|
|
70
|
+
) if on]
|
|
71
|
+
if insecure and not settings.DEBUG:
|
|
72
|
+
out.append(Warning(
|
|
73
|
+
f"{', '.join(insecure)} enabled with DEBUG=False: personal or secret data may be stored "
|
|
74
|
+
"(values are redacted by key/pattern, but review REDACTION for your data).", id="observability.W004"))
|
|
75
|
+
if cfg["PIPELINE"]["SYNC"] and not settings.DEBUG:
|
|
76
|
+
out.append(Warning("PIPELINE['SYNC'] writes telemetry inside the request; use only for tests.",
|
|
77
|
+
id="observability.W005"))
|
|
78
|
+
if cfg["OTEL"].get("enabled") and not cfg["OTEL"].get("endpoint"):
|
|
79
|
+
out.append(Error("OTEL is enabled but OTEL['endpoint'] is empty.", id="observability.E009"))
|
|
80
|
+
if cfg["PIPELINE"]["QUEUE_SIZE"] < 100:
|
|
81
|
+
out.append(Error("PIPELINE['QUEUE_SIZE'] must be at least 100.", id="observability.E010"))
|
|
82
|
+
return out
|