setu-trafficmonitor 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- setu_trafficmonitor-2.0.0.dist-info/LICENSE +21 -0
- setu_trafficmonitor-2.0.0.dist-info/METADATA +401 -0
- setu_trafficmonitor-2.0.0.dist-info/RECORD +42 -0
- setu_trafficmonitor-2.0.0.dist-info/WHEEL +5 -0
- setu_trafficmonitor-2.0.0.dist-info/top_level.txt +1 -0
- trafficmonitor/__init__.py +11 -0
- trafficmonitor/admin.py +217 -0
- trafficmonitor/analytics/__init__.py +0 -0
- trafficmonitor/analytics/enhanced_queries.py +286 -0
- trafficmonitor/analytics/serializers.py +238 -0
- trafficmonitor/analytics/tests.py +757 -0
- trafficmonitor/analytics/urls.py +18 -0
- trafficmonitor/analytics/views.py +694 -0
- trafficmonitor/apps.py +7 -0
- trafficmonitor/circuit_breaker.py +63 -0
- trafficmonitor/conf.py +154 -0
- trafficmonitor/dashboard_security.py +111 -0
- trafficmonitor/db_utils.py +37 -0
- trafficmonitor/exceptions.py +93 -0
- trafficmonitor/health.py +66 -0
- trafficmonitor/load_test.py +423 -0
- trafficmonitor/load_test_api.py +307 -0
- trafficmonitor/management/__init__.py +1 -0
- trafficmonitor/management/commands/__init__.py +1 -0
- trafficmonitor/management/commands/cleanup_request_logs.py +77 -0
- trafficmonitor/middleware.py +383 -0
- trafficmonitor/migrations/0001_initial.py +93 -0
- trafficmonitor/migrations/__init__.py +0 -0
- trafficmonitor/models.py +206 -0
- trafficmonitor/monitoring.py +104 -0
- trafficmonitor/permissions.py +64 -0
- trafficmonitor/security.py +180 -0
- trafficmonitor/settings_production.py +105 -0
- trafficmonitor/static/analytics/css/dashboard.css +99 -0
- trafficmonitor/static/analytics/js/dashboard-production.js +339 -0
- trafficmonitor/static/analytics/js/dashboard-v2.js +697 -0
- trafficmonitor/static/analytics/js/dashboard.js +693 -0
- trafficmonitor/tasks.py +137 -0
- trafficmonitor/templates/analytics/dashboard.html +500 -0
- trafficmonitor/tests.py +246 -0
- trafficmonitor/views.py +3 -0
- trafficmonitor/websocket_consumers.py +128 -0
|
@@ -0,0 +1,383 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
import time
|
|
4
|
+
import traceback
|
|
5
|
+
import random
|
|
6
|
+
from io import BytesIO
|
|
7
|
+
from threading import local
|
|
8
|
+
from contextlib import contextmanager
|
|
9
|
+
|
|
10
|
+
from django.db import connection, transaction
|
|
11
|
+
from django.utils.deprecation import MiddlewareMixin
|
|
12
|
+
from django.core.cache import cache
|
|
13
|
+
|
|
14
|
+
from trafficmonitor.models import RequestLog
|
|
15
|
+
from trafficmonitor.conf import TrafficMonitorConfig
|
|
16
|
+
from trafficmonitor.circuit_breaker import CircuitBreaker
|
|
17
|
+
|
|
18
|
+
logger = logging.getLogger(__name__)
|
|
19
|
+
|
|
20
|
+
# Thread-local storage for request context
|
|
21
|
+
_local = local()
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class RequestLoggingMiddleware(MiddlewareMixin):
|
|
25
|
+
"""
|
|
26
|
+
Enterprise-grade middleware for HTTP request logging.
|
|
27
|
+
Features: Circuit breaker, async logging, PII sanitization, metrics collection.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
EXCLUDED_PATHS = [
|
|
31
|
+
'/admin/jsi18n/',
|
|
32
|
+
'/static/',
|
|
33
|
+
'/media/',
|
|
34
|
+
'/__debug__/',
|
|
35
|
+
'/favicon.ico',
|
|
36
|
+
'/health/',
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
def __init__(self, get_response):
|
|
40
|
+
super().__init__(get_response)
|
|
41
|
+
self.circuit_breaker = CircuitBreaker() if TrafficMonitorConfig.CIRCUIT_BREAKER_ENABLED else None
|
|
42
|
+
|
|
43
|
+
def process_request(self, request):
|
|
44
|
+
"""Initialize request tracking"""
|
|
45
|
+
request._logging_start_time = time.time()
|
|
46
|
+
request._logging_query_count_start = len(connection.queries)
|
|
47
|
+
return None
|
|
48
|
+
|
|
49
|
+
def process_response(self, request, response):
|
|
50
|
+
"""Log request with enterprise features"""
|
|
51
|
+
try:
|
|
52
|
+
# Skip excluded paths
|
|
53
|
+
if self._should_exclude_path(request.path):
|
|
54
|
+
return response
|
|
55
|
+
|
|
56
|
+
# Apply sampling
|
|
57
|
+
if TrafficMonitorConfig.ENABLE_SAMPLING and random.random() > TrafficMonitorConfig.SAMPLE_RATE:
|
|
58
|
+
return response
|
|
59
|
+
|
|
60
|
+
# Collect metrics
|
|
61
|
+
if TrafficMonitorConfig.ENABLE_METRICS:
|
|
62
|
+
self._collect_metrics(request, response)
|
|
63
|
+
|
|
64
|
+
# Log request
|
|
65
|
+
if self.circuit_breaker:
|
|
66
|
+
self.circuit_breaker.call(self._log_request_safe, request, response)
|
|
67
|
+
else:
|
|
68
|
+
self._log_request_safe(request, response)
|
|
69
|
+
|
|
70
|
+
except Exception as e:
|
|
71
|
+
# Never let logging break the request
|
|
72
|
+
logger.error(f"Request logging failed: {e}", exc_info=True)
|
|
73
|
+
|
|
74
|
+
return response
|
|
75
|
+
|
|
76
|
+
def _log_request_safe(self, request, response):
|
|
77
|
+
"""Safely log request with error handling"""
|
|
78
|
+
from trafficmonitor.security import DataSanitizer, SecurityValidator
|
|
79
|
+
from trafficmonitor.monitoring import performance_timer
|
|
80
|
+
|
|
81
|
+
with performance_timer("request_logging_duration"):
|
|
82
|
+
log_data = self._build_log_data(request, response)
|
|
83
|
+
|
|
84
|
+
if TrafficMonitorConfig.ASYNC_LOGGING:
|
|
85
|
+
self._log_async(log_data)
|
|
86
|
+
else:
|
|
87
|
+
self._log_sync(log_data)
|
|
88
|
+
|
|
89
|
+
def _build_log_data(self, request, response):
|
|
90
|
+
"""Build sanitized log data"""
|
|
91
|
+
from trafficmonitor.security import DataSanitizer, SecurityValidator
|
|
92
|
+
|
|
93
|
+
# Calculate metrics
|
|
94
|
+
duration_ms = (time.time() - request._logging_start_time) * 1000
|
|
95
|
+
query_count = len(connection.queries) - request._logging_query_count_start
|
|
96
|
+
|
|
97
|
+
# Get user info
|
|
98
|
+
user_info = TrafficMonitorConfig.get_user_info_from_request(request)
|
|
99
|
+
|
|
100
|
+
# Sanitize data
|
|
101
|
+
headers = DataSanitizer.sanitize_headers(dict(request.META))
|
|
102
|
+
body = self._get_request_body(request)
|
|
103
|
+
if TrafficMonitorConfig.ENABLE_PII_SANITIZATION:
|
|
104
|
+
body = DataSanitizer.sanitize_body(body, TrafficMonitorConfig.MAX_BODY_LENGTH)
|
|
105
|
+
|
|
106
|
+
return {
|
|
107
|
+
'method': request.method,
|
|
108
|
+
'path': request.path,
|
|
109
|
+
'full_url': request.build_absolute_uri(),
|
|
110
|
+
'status_code': response.status_code,
|
|
111
|
+
'requested_user_id': user_info.get('user_id'),
|
|
112
|
+
'ip_address': SecurityValidator.get_client_ip(request),
|
|
113
|
+
'user_agent': request.META.get('HTTP_USER_AGENT', ''),
|
|
114
|
+
'request_headers': headers,
|
|
115
|
+
'request_body': body,
|
|
116
|
+
'operation_type': RequestLog.get_operation_type(request.method),
|
|
117
|
+
'endpoint_category': RequestLog.extract_endpoint_category(request.path),
|
|
118
|
+
'is_api_request': request.path.startswith('/api/'),
|
|
119
|
+
'response_time_ms': duration_ms,
|
|
120
|
+
'query_count': query_count,
|
|
121
|
+
'content_length': len(response.content) if hasattr(response, 'content') else None,
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
def _collect_metrics(self, request, response):
|
|
125
|
+
"""Collect application metrics"""
|
|
126
|
+
from trafficmonitor.monitoring import metrics
|
|
127
|
+
|
|
128
|
+
tags = {
|
|
129
|
+
'method': request.method,
|
|
130
|
+
'status_code': str(response.status_code),
|
|
131
|
+
'endpoint': request.path[:50] # Truncate long paths
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
metrics.increment('requests_total', tags=tags)
|
|
135
|
+
|
|
136
|
+
if response.status_code >= 400:
|
|
137
|
+
metrics.increment('requests_errors', tags=tags)
|
|
138
|
+
|
|
139
|
+
if response.status_code >= 500:
|
|
140
|
+
metrics.increment('requests_server_errors', tags=tags)
|
|
141
|
+
|
|
142
|
+
def _log_async(self, log_data):
|
|
143
|
+
"""Log asynchronously using Celery"""
|
|
144
|
+
try:
|
|
145
|
+
from trafficmonitor.tasks import log_request_async
|
|
146
|
+
log_request_async.delay(log_data)
|
|
147
|
+
except ImportError:
|
|
148
|
+
logger.warning("Celery not available, falling back to sync logging")
|
|
149
|
+
self._log_sync(log_data)
|
|
150
|
+
|
|
151
|
+
def _log_sync(self, log_data):
|
|
152
|
+
"""Log synchronously"""
|
|
153
|
+
try:
|
|
154
|
+
RequestLog.objects.create(**log_data)
|
|
155
|
+
except Exception as e:
|
|
156
|
+
logger.error(f"Failed to create RequestLog: {e}", exc_info=True)
|
|
157
|
+
|
|
158
|
+
def _should_exclude_path(self, path):
|
|
159
|
+
"""Check if path should be excluded from logging"""
|
|
160
|
+
return any(excluded in path for excluded in TrafficMonitorConfig.EXCLUDED_PATHS)
|
|
161
|
+
|
|
162
|
+
def _get_request_body(self, request):
|
|
163
|
+
"""Safely get request body"""
|
|
164
|
+
try:
|
|
165
|
+
if hasattr(request, '_body'):
|
|
166
|
+
return request._body.decode('utf-8')
|
|
167
|
+
elif hasattr(request, 'body'):
|
|
168
|
+
return request.body.decode('utf-8')
|
|
169
|
+
except (UnicodeDecodeError, AttributeError):
|
|
170
|
+
pass
|
|
171
|
+
return ""
|
|
172
|
+
|
|
173
|
+
try:
|
|
174
|
+
# Calculate response time
|
|
175
|
+
response_time_ms = None
|
|
176
|
+
if hasattr(request, '_logging_start_time'):
|
|
177
|
+
response_time_ms = (time.time() - request._logging_start_time) * 1000
|
|
178
|
+
|
|
179
|
+
# Calculate query count
|
|
180
|
+
query_count = None
|
|
181
|
+
if hasattr(request, '_logging_query_count_start'):
|
|
182
|
+
query_count = len(connection.queries) - request._logging_query_count_start
|
|
183
|
+
|
|
184
|
+
# Extract request data
|
|
185
|
+
request_headers = self._get_request_headers(request)
|
|
186
|
+
request_body = self._get_request_body(request)
|
|
187
|
+
|
|
188
|
+
# Get user ID from header (X-User-ID)
|
|
189
|
+
requested_user_id = request.META.get('HTTP_X_USER_ID') or request.headers.get('X-User-ID')
|
|
190
|
+
|
|
191
|
+
# Get IP address
|
|
192
|
+
ip_address = self._get_client_ip(request)
|
|
193
|
+
|
|
194
|
+
# Determine operation type and endpoint category
|
|
195
|
+
operation_type = RequestLog.get_operation_type(request.method)
|
|
196
|
+
endpoint_category = RequestLog.extract_endpoint_category(request.path)
|
|
197
|
+
is_api_request = request.path.startswith('/api/')
|
|
198
|
+
|
|
199
|
+
# Prepare log data
|
|
200
|
+
log_data = {
|
|
201
|
+
'method': request.method,
|
|
202
|
+
'path': request.path,
|
|
203
|
+
'full_url': request.build_absolute_uri(),
|
|
204
|
+
'status_code': response.status_code,
|
|
205
|
+
'requested_user_id': requested_user_id,
|
|
206
|
+
'ip_address': ip_address,
|
|
207
|
+
'user_agent': request.META.get('HTTP_USER_AGENT', ''),
|
|
208
|
+
'request_headers': request_headers,
|
|
209
|
+
'request_body': request_body,
|
|
210
|
+
'operation_type': operation_type,
|
|
211
|
+
'endpoint_category': endpoint_category,
|
|
212
|
+
'is_api_request': is_api_request,
|
|
213
|
+
'response_time_ms': response_time_ms,
|
|
214
|
+
'query_count': query_count,
|
|
215
|
+
'content_length': len(response.content) if hasattr(response, 'content') else None,
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
# Log asynchronously or synchronously based on config
|
|
219
|
+
if TrafficMonitorConfig.ASYNC_LOGGING:
|
|
220
|
+
self._log_async(log_data)
|
|
221
|
+
else:
|
|
222
|
+
RequestLog.objects.create(**log_data)
|
|
223
|
+
|
|
224
|
+
except Exception as e:
|
|
225
|
+
# Log the error but don't break the request/response cycle
|
|
226
|
+
logger.error(f"Error logging request: {str(e)}", exc_info=True)
|
|
227
|
+
|
|
228
|
+
return response
|
|
229
|
+
|
|
230
|
+
def process_exception(self, request, exception):
|
|
231
|
+
"""
|
|
232
|
+
Called when a view raises an exception.
|
|
233
|
+
Logs the exception details.
|
|
234
|
+
"""
|
|
235
|
+
# Skip logging for excluded paths
|
|
236
|
+
if self._should_exclude_path(request.path):
|
|
237
|
+
return None
|
|
238
|
+
|
|
239
|
+
try:
|
|
240
|
+
# Calculate response time
|
|
241
|
+
response_time_ms = None
|
|
242
|
+
if hasattr(request, '_logging_start_time'):
|
|
243
|
+
response_time_ms = (time.time() - request._logging_start_time) * 1000
|
|
244
|
+
|
|
245
|
+
# Calculate query count
|
|
246
|
+
query_count = None
|
|
247
|
+
if hasattr(request, '_logging_query_count_start'):
|
|
248
|
+
query_count = len(connection.queries) - request._logging_query_count_start
|
|
249
|
+
|
|
250
|
+
# Extract request data
|
|
251
|
+
request_headers = self._get_request_headers(request)
|
|
252
|
+
request_body = self._get_request_body(request)
|
|
253
|
+
|
|
254
|
+
# Get user ID from header
|
|
255
|
+
requested_user_id = request.META.get('HTTP_X_USER_ID') or request.headers.get('X-User-ID')
|
|
256
|
+
|
|
257
|
+
# Get IP address
|
|
258
|
+
ip_address = self._get_client_ip(request)
|
|
259
|
+
|
|
260
|
+
# Determine operation type and endpoint category
|
|
261
|
+
operation_type = RequestLog.get_operation_type(request.method)
|
|
262
|
+
endpoint_category = RequestLog.extract_endpoint_category(request.path)
|
|
263
|
+
is_api_request = request.path.startswith('/api/')
|
|
264
|
+
|
|
265
|
+
# Get exception traceback
|
|
266
|
+
exception_traceback = ''.join(traceback.format_exception(
|
|
267
|
+
type(exception), exception, exception.__traceback__
|
|
268
|
+
))
|
|
269
|
+
|
|
270
|
+
# Prepare log data with exception
|
|
271
|
+
log_data = {
|
|
272
|
+
'method': request.method,
|
|
273
|
+
'path': request.path,
|
|
274
|
+
'full_url': request.build_absolute_uri(),
|
|
275
|
+
'status_code': 500, # Internal Server Error
|
|
276
|
+
'requested_user_id': requested_user_id,
|
|
277
|
+
'ip_address': ip_address,
|
|
278
|
+
'user_agent': request.META.get('HTTP_USER_AGENT', ''),
|
|
279
|
+
'request_headers': request_headers,
|
|
280
|
+
'request_body': request_body,
|
|
281
|
+
'operation_type': operation_type,
|
|
282
|
+
'endpoint_category': endpoint_category,
|
|
283
|
+
'is_api_request': is_api_request,
|
|
284
|
+
'response_time_ms': response_time_ms,
|
|
285
|
+
'query_count': query_count,
|
|
286
|
+
'exception': exception_traceback,
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
# Log asynchronously or synchronously
|
|
290
|
+
if TrafficMonitorConfig.ASYNC_LOGGING:
|
|
291
|
+
self._log_async(log_data)
|
|
292
|
+
else:
|
|
293
|
+
RequestLog.objects.create(**log_data)
|
|
294
|
+
|
|
295
|
+
except Exception as e:
|
|
296
|
+
# Log the error but don't break the request/response cycle
|
|
297
|
+
logger.error(f"Error logging exception: {str(e)}", exc_info=True)
|
|
298
|
+
|
|
299
|
+
return None
|
|
300
|
+
|
|
301
|
+
def _should_exclude_path(self, path):
|
|
302
|
+
"""
|
|
303
|
+
Check if the request path should be excluded from logging.
|
|
304
|
+
"""
|
|
305
|
+
for excluded_path in self.EXCLUDED_PATHS:
|
|
306
|
+
if path.startswith(excluded_path):
|
|
307
|
+
return True
|
|
308
|
+
return False
|
|
309
|
+
|
|
310
|
+
def _log_async(self, log_data):
|
|
311
|
+
"""
|
|
312
|
+
Log request data asynchronously using Celery.
|
|
313
|
+
Falls back to synchronous logging if Celery is unavailable.
|
|
314
|
+
"""
|
|
315
|
+
try:
|
|
316
|
+
from trafficmonitor.tasks import log_request_async
|
|
317
|
+
log_request_async.delay(log_data)
|
|
318
|
+
except Exception as e:
|
|
319
|
+
logger.warning(f"Async logging failed, falling back to sync: {e}")
|
|
320
|
+
RequestLog.objects.create(**log_data)
|
|
321
|
+
|
|
322
|
+
def _get_request_body(self, request):
|
|
323
|
+
"""
|
|
324
|
+
Extract and return the request body.
|
|
325
|
+
Handles both JSON and form data.
|
|
326
|
+
"""
|
|
327
|
+
try:
|
|
328
|
+
if request.method in ['POST', 'PUT', 'PATCH']:
|
|
329
|
+
# Try to get JSON body
|
|
330
|
+
if request.content_type and 'application/json' in request.content_type:
|
|
331
|
+
body = request.body.decode('utf-8')
|
|
332
|
+
# Truncate if too long
|
|
333
|
+
if len(body) > self.MAX_BODY_LENGTH:
|
|
334
|
+
body = body[:self.MAX_BODY_LENGTH] + '... [truncated]'
|
|
335
|
+
return body
|
|
336
|
+
# For form data, log the POST data
|
|
337
|
+
elif request.POST:
|
|
338
|
+
body = json.dumps(dict(request.POST))
|
|
339
|
+
if len(body) > self.MAX_BODY_LENGTH:
|
|
340
|
+
body = body[:self.MAX_BODY_LENGTH] + '... [truncated]'
|
|
341
|
+
return body
|
|
342
|
+
except Exception as e:
|
|
343
|
+
logger.debug(f"Error extracting request body: {str(e)}")
|
|
344
|
+
|
|
345
|
+
return None
|
|
346
|
+
|
|
347
|
+
def _get_request_headers(self, request):
|
|
348
|
+
"""
|
|
349
|
+
Extract and return request headers as a dictionary.
|
|
350
|
+
Filters out sensitive headers and ensures JSON serializable values.
|
|
351
|
+
"""
|
|
352
|
+
sensitive_headers = [
|
|
353
|
+
'HTTP_AUTHORIZATION',
|
|
354
|
+
'HTTP_COOKIE',
|
|
355
|
+
'HTTP_X_CSRFTOKEN',
|
|
356
|
+
]
|
|
357
|
+
|
|
358
|
+
headers = {}
|
|
359
|
+
for key, value in request.META.items():
|
|
360
|
+
if key.startswith('HTTP_') and key not in sensitive_headers:
|
|
361
|
+
# Remove HTTP_ prefix and convert to title case
|
|
362
|
+
header_name = key[5:].replace('_', '-').title()
|
|
363
|
+
# Ensure value is JSON serializable
|
|
364
|
+
try:
|
|
365
|
+
headers[header_name] = str(value)
|
|
366
|
+
except:
|
|
367
|
+
headers[header_name] = '<non-serializable>'
|
|
368
|
+
|
|
369
|
+
return headers
|
|
370
|
+
|
|
371
|
+
def _get_client_ip(self, request):
|
|
372
|
+
"""
|
|
373
|
+
Extract and return the client's IP address.
|
|
374
|
+
Handles X-Forwarded-For header for proxied requests.
|
|
375
|
+
"""
|
|
376
|
+
x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
|
|
377
|
+
if x_forwarded_for:
|
|
378
|
+
# Get the first IP in the chain
|
|
379
|
+
ip = x_forwarded_for.split(',')[0].strip()
|
|
380
|
+
else:
|
|
381
|
+
ip = request.META.get('REMOTE_ADDR')
|
|
382
|
+
|
|
383
|
+
return ip
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# Generated by Django 3.2.10 on 2025-12-22 10:52
|
|
2
|
+
|
|
3
|
+
import django.core.serializers.json
|
|
4
|
+
from django.db import migrations, models
|
|
5
|
+
import uuid
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Migration(migrations.Migration):
|
|
9
|
+
|
|
10
|
+
initial = True
|
|
11
|
+
|
|
12
|
+
dependencies = [
|
|
13
|
+
]
|
|
14
|
+
|
|
15
|
+
operations = [
|
|
16
|
+
migrations.CreateModel(
|
|
17
|
+
name='RequestLog',
|
|
18
|
+
fields=[
|
|
19
|
+
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
|
20
|
+
('method', models.CharField(db_index=True, help_text='HTTP method (GET, POST, etc.)', max_length=10)),
|
|
21
|
+
('path', models.CharField(db_index=True, help_text='Request path/URL', max_length=2048)),
|
|
22
|
+
('full_url', models.TextField(help_text='Complete URL including query parameters')),
|
|
23
|
+
('status_code', models.IntegerField(db_index=True, help_text='HTTP response status code')),
|
|
24
|
+
('requested_user_id', models.CharField(blank=True, db_index=True, help_text='User ID from request header (X-User-ID)', max_length=255, null=True)),
|
|
25
|
+
('ip_address', models.GenericIPAddressField(blank=True, db_index=True, help_text='Client IP address', null=True)),
|
|
26
|
+
('user_agent', models.TextField(blank=True, help_text='User agent string from request headers', null=True)),
|
|
27
|
+
('request_headers', models.JSONField(blank=True, encoder=django.core.serializers.json.DjangoJSONEncoder, help_text='Request headers as JSON (sensitive headers excluded)', null=True)),
|
|
28
|
+
('request_body', models.TextField(blank=True, help_text='Request body content (truncated for large payloads)', null=True)),
|
|
29
|
+
('operation_type', models.CharField(blank=True, choices=[('READ', 'Read Operation'), ('WRITE', 'Write Operation'), ('DELETE', 'Delete Operation')], db_index=True, help_text='Operation type: READ (GET, HEAD, OPTIONS), WRITE (POST, PUT, PATCH), DELETE', max_length=10, null=True)),
|
|
30
|
+
('endpoint_category', models.CharField(blank=True, db_index=True, help_text='Endpoint category extracted from path (e.g., /api/users -> users)', max_length=100, null=True)),
|
|
31
|
+
('is_api_request', models.BooleanField(db_index=True, default=False, help_text='Whether this is an API request (starts with /api/)')),
|
|
32
|
+
('response_time_ms', models.FloatField(blank=True, db_index=True, help_text='Response time in milliseconds', null=True)),
|
|
33
|
+
('query_count', models.IntegerField(blank=True, help_text='Number of database queries executed', null=True)),
|
|
34
|
+
('exception', models.TextField(blank=True, help_text='Exception traceback if request failed', null=True)),
|
|
35
|
+
('content_length', models.IntegerField(blank=True, help_text='Response content length in bytes', null=True)),
|
|
36
|
+
('timestamp', models.DateTimeField(auto_now_add=True, db_index=True)),
|
|
37
|
+
],
|
|
38
|
+
options={
|
|
39
|
+
'verbose_name': 'Request Log',
|
|
40
|
+
'verbose_name_plural': 'Request Logs',
|
|
41
|
+
'db_table': 'request_log',
|
|
42
|
+
'ordering': ['-timestamp'],
|
|
43
|
+
},
|
|
44
|
+
),
|
|
45
|
+
migrations.AddIndex(
|
|
46
|
+
model_name='requestlog',
|
|
47
|
+
index=models.Index(fields=['-timestamp', 'status_code'], name='idx_timestamp_status'),
|
|
48
|
+
),
|
|
49
|
+
migrations.AddIndex(
|
|
50
|
+
model_name='requestlog',
|
|
51
|
+
index=models.Index(fields=['requested_user_id', '-timestamp'], name='idx_user_timestamp'),
|
|
52
|
+
),
|
|
53
|
+
migrations.AddIndex(
|
|
54
|
+
model_name='requestlog',
|
|
55
|
+
index=models.Index(fields=['path', '-timestamp'], name='idx_path_timestamp'),
|
|
56
|
+
),
|
|
57
|
+
migrations.AddIndex(
|
|
58
|
+
model_name='requestlog',
|
|
59
|
+
index=models.Index(fields=['operation_type', '-timestamp'], name='idx_operation_timestamp'),
|
|
60
|
+
),
|
|
61
|
+
migrations.AddIndex(
|
|
62
|
+
model_name='requestlog',
|
|
63
|
+
index=models.Index(fields=['endpoint_category', '-timestamp'], name='idx_category_timestamp'),
|
|
64
|
+
),
|
|
65
|
+
migrations.AddIndex(
|
|
66
|
+
model_name='requestlog',
|
|
67
|
+
index=models.Index(fields=['is_api_request', 'status_code', '-timestamp'], name='idx_api_status_timestamp'),
|
|
68
|
+
),
|
|
69
|
+
migrations.AddIndex(
|
|
70
|
+
model_name='requestlog',
|
|
71
|
+
index=models.Index(fields=['method', 'status_code', '-timestamp'], name='idx_method_status_timestamp'),
|
|
72
|
+
),
|
|
73
|
+
migrations.AddIndex(
|
|
74
|
+
model_name='requestlog',
|
|
75
|
+
index=models.Index(fields=['response_time_ms', '-timestamp'], name='idx_response_time'),
|
|
76
|
+
),
|
|
77
|
+
migrations.AddIndex(
|
|
78
|
+
model_name='requestlog',
|
|
79
|
+
index=models.Index(fields=['query_count', '-timestamp'], name='idx_query_count'),
|
|
80
|
+
),
|
|
81
|
+
migrations.AddIndex(
|
|
82
|
+
model_name='requestlog',
|
|
83
|
+
index=models.Index(condition=models.Q(('status_code__gte', 400)), fields=['status_code', '-timestamp'], name='idx_errors_only'),
|
|
84
|
+
),
|
|
85
|
+
migrations.AddConstraint(
|
|
86
|
+
model_name='requestlog',
|
|
87
|
+
constraint=models.CheckConstraint(check=models.Q(('status_code__gte', 100), ('status_code__lt', 600)), name='valid_status_code'),
|
|
88
|
+
),
|
|
89
|
+
migrations.AddConstraint(
|
|
90
|
+
model_name='requestlog',
|
|
91
|
+
constraint=models.CheckConstraint(check=models.Q(('response_time_ms__gte', 0)), name='positive_response_time'),
|
|
92
|
+
),
|
|
93
|
+
]
|
|
File without changes
|
trafficmonitor/models.py
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
import uuid
|
|
2
|
+
from django.conf import settings
|
|
3
|
+
from django.db import models
|
|
4
|
+
from django.core.serializers.json import DjangoJSONEncoder
|
|
5
|
+
from django.utils import timezone
|
|
6
|
+
from datetime import timedelta
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class RequestLogManager(models.Manager):
|
|
10
|
+
"""Custom manager for RequestLog with optimized queries"""
|
|
11
|
+
|
|
12
|
+
def get_recent_logs(self, days: int = 7):
|
|
13
|
+
"""Get logs from the last N days"""
|
|
14
|
+
cutoff_date = timezone.now() - timedelta(days=days)
|
|
15
|
+
return self.filter(timestamp__gte=cutoff_date)
|
|
16
|
+
|
|
17
|
+
def get_error_logs(self, days: int = 1):
|
|
18
|
+
"""Get error logs (4xx, 5xx) from the last N days"""
|
|
19
|
+
cutoff_date = timezone.now() - timedelta(days=days)
|
|
20
|
+
return self.filter(
|
|
21
|
+
timestamp__gte=cutoff_date,
|
|
22
|
+
status_code__gte=400
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
def cleanup_old_logs(self, retention_days: int = 90):
|
|
26
|
+
"""Delete logs older than retention period"""
|
|
27
|
+
cutoff_date = timezone.now() - timedelta(days=retention_days)
|
|
28
|
+
deleted_count, _ = self.filter(timestamp__lt=cutoff_date).delete()
|
|
29
|
+
return deleted_count
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class RequestLog(models.Model):
|
|
33
|
+
"""
|
|
34
|
+
Enterprise-grade model for HTTP request logging.
|
|
35
|
+
Optimized for high-volume traffic with partitioning support.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
# Operation Types
|
|
39
|
+
OPERATION_READ = 'READ'
|
|
40
|
+
OPERATION_WRITE = 'WRITE'
|
|
41
|
+
OPERATION_DELETE = 'DELETE'
|
|
42
|
+
|
|
43
|
+
OPERATION_CHOICES = [
|
|
44
|
+
(OPERATION_READ, 'Read Operation'),
|
|
45
|
+
(OPERATION_WRITE, 'Write Operation'),
|
|
46
|
+
(OPERATION_DELETE, 'Delete Operation'),
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
|
|
50
|
+
|
|
51
|
+
# Basic Request Info
|
|
52
|
+
method = models.CharField(max_length=10, db_index=True, help_text="HTTP method (GET, POST, etc.)")
|
|
53
|
+
path = models.CharField(max_length=2048, db_index=True, help_text="Request path/URL")
|
|
54
|
+
full_url = models.TextField(help_text="Complete URL including query parameters")
|
|
55
|
+
|
|
56
|
+
# Response Info
|
|
57
|
+
status_code = models.IntegerField(db_index=True, help_text="HTTP response status code")
|
|
58
|
+
|
|
59
|
+
# User & IP Tracking (Enterprise: Header-based user ID)
|
|
60
|
+
requested_user_id = models.CharField(
|
|
61
|
+
max_length=255,
|
|
62
|
+
null=True,
|
|
63
|
+
blank=True,
|
|
64
|
+
db_index=True,
|
|
65
|
+
help_text="User ID from request header (X-User-ID)"
|
|
66
|
+
)
|
|
67
|
+
ip_address = models.GenericIPAddressField(
|
|
68
|
+
null=True,
|
|
69
|
+
blank=True,
|
|
70
|
+
db_index=True,
|
|
71
|
+
help_text="Client IP address"
|
|
72
|
+
)
|
|
73
|
+
user_agent = models.TextField(
|
|
74
|
+
null=True,
|
|
75
|
+
blank=True,
|
|
76
|
+
help_text="User agent string from request headers"
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
# Request Metadata
|
|
80
|
+
request_headers = models.JSONField(
|
|
81
|
+
null=True,
|
|
82
|
+
blank=True,
|
|
83
|
+
encoder=DjangoJSONEncoder,
|
|
84
|
+
help_text="Request headers as JSON (sensitive headers excluded)"
|
|
85
|
+
)
|
|
86
|
+
request_body = models.TextField(
|
|
87
|
+
null=True,
|
|
88
|
+
blank=True,
|
|
89
|
+
help_text="Request body content (truncated for large payloads)"
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
# Operation Classification
|
|
93
|
+
operation_type = models.CharField(
|
|
94
|
+
max_length=10,
|
|
95
|
+
choices=OPERATION_CHOICES,
|
|
96
|
+
null=True,
|
|
97
|
+
blank=True,
|
|
98
|
+
db_index=True,
|
|
99
|
+
help_text="Operation type: READ (GET, HEAD, OPTIONS), WRITE (POST, PUT, PATCH), DELETE"
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
endpoint_category = models.CharField(
|
|
103
|
+
max_length=100,
|
|
104
|
+
null=True,
|
|
105
|
+
blank=True,
|
|
106
|
+
db_index=True,
|
|
107
|
+
help_text="Endpoint category extracted from path (e.g., /api/users -> users)"
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
is_api_request = models.BooleanField(
|
|
111
|
+
default=False,
|
|
112
|
+
db_index=True,
|
|
113
|
+
help_text="Whether this is an API request (starts with /api/)"
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
# Performance Metrics
|
|
117
|
+
response_time_ms = models.FloatField(
|
|
118
|
+
null=True,
|
|
119
|
+
blank=True,
|
|
120
|
+
db_index=True,
|
|
121
|
+
help_text="Response time in milliseconds"
|
|
122
|
+
)
|
|
123
|
+
query_count = models.IntegerField(
|
|
124
|
+
null=True,
|
|
125
|
+
blank=True,
|
|
126
|
+
help_text="Number of database queries executed"
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
# Error Tracking
|
|
130
|
+
exception = models.TextField(
|
|
131
|
+
null=True,
|
|
132
|
+
blank=True,
|
|
133
|
+
help_text="Exception traceback if request failed"
|
|
134
|
+
)
|
|
135
|
+
content_length = models.IntegerField(
|
|
136
|
+
null=True,
|
|
137
|
+
blank=True,
|
|
138
|
+
help_text="Response content length in bytes"
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
# Timestamps
|
|
142
|
+
timestamp = models.DateTimeField(auto_now_add=True, db_index=True)
|
|
143
|
+
|
|
144
|
+
# Custom manager
|
|
145
|
+
objects = RequestLogManager()
|
|
146
|
+
|
|
147
|
+
class Meta:
|
|
148
|
+
db_table = "request_log"
|
|
149
|
+
verbose_name = "Request Log"
|
|
150
|
+
verbose_name_plural = "Request Logs"
|
|
151
|
+
ordering = ["-timestamp"]
|
|
152
|
+
|
|
153
|
+
# Optimized indexes for enterprise queries
|
|
154
|
+
indexes = [
|
|
155
|
+
# Primary query patterns
|
|
156
|
+
models.Index(fields=["-timestamp", "status_code"], name="idx_timestamp_status"),
|
|
157
|
+
models.Index(fields=["requested_user_id", "-timestamp"], name="idx_user_timestamp"),
|
|
158
|
+
models.Index(fields=["path", "-timestamp"], name="idx_path_timestamp"),
|
|
159
|
+
models.Index(fields=["operation_type", "-timestamp"], name="idx_operation_timestamp"),
|
|
160
|
+
models.Index(fields=["endpoint_category", "-timestamp"], name="idx_category_timestamp"),
|
|
161
|
+
|
|
162
|
+
# Analytics queries
|
|
163
|
+
models.Index(fields=["is_api_request", "status_code", "-timestamp"], name="idx_api_status_timestamp"),
|
|
164
|
+
models.Index(fields=["method", "status_code", "-timestamp"], name="idx_method_status_timestamp"),
|
|
165
|
+
|
|
166
|
+
# Performance monitoring
|
|
167
|
+
models.Index(fields=["response_time_ms", "-timestamp"], name="idx_response_time"),
|
|
168
|
+
models.Index(fields=["query_count", "-timestamp"], name="idx_query_count"),
|
|
169
|
+
|
|
170
|
+
# Error tracking
|
|
171
|
+
models.Index(fields=["status_code", "-timestamp"], condition=models.Q(status_code__gte=400), name="idx_errors_only"),
|
|
172
|
+
]
|
|
173
|
+
|
|
174
|
+
# Database constraints
|
|
175
|
+
constraints = [
|
|
176
|
+
models.CheckConstraint(
|
|
177
|
+
check=models.Q(status_code__gte=100) & models.Q(status_code__lt=600),
|
|
178
|
+
name="valid_status_code"
|
|
179
|
+
),
|
|
180
|
+
models.CheckConstraint(
|
|
181
|
+
check=models.Q(response_time_ms__gte=0),
|
|
182
|
+
name="positive_response_time"
|
|
183
|
+
),
|
|
184
|
+
]
|
|
185
|
+
|
|
186
|
+
def __str__(self):
|
|
187
|
+
user_str = self.requested_user_id or "Anonymous"
|
|
188
|
+
op_type = f"[{self.operation_type}]" if self.operation_type else ""
|
|
189
|
+
return f"{self.method} {self.path} [{self.status_code}] {op_type} - {user_str} at {self.timestamp}"
|
|
190
|
+
|
|
191
|
+
@staticmethod
|
|
192
|
+
def get_operation_type(method):
|
|
193
|
+
"""Determine operation type from HTTP method"""
|
|
194
|
+
if method in ['GET', 'HEAD', 'OPTIONS']:
|
|
195
|
+
return RequestLog.OPERATION_READ
|
|
196
|
+
elif method == 'DELETE':
|
|
197
|
+
return RequestLog.OPERATION_DELETE
|
|
198
|
+
elif method in ['POST', 'PUT', 'PATCH']:
|
|
199
|
+
return RequestLog.OPERATION_WRITE
|
|
200
|
+
return None
|
|
201
|
+
|
|
202
|
+
@staticmethod
|
|
203
|
+
def extract_endpoint_category(path):
|
|
204
|
+
"""Extract category from path (e.g., /api/users/123 -> users)"""
|
|
205
|
+
parts = [p for p in path.split('/') if p and p != 'api']
|
|
206
|
+
return parts[0] if parts else None
|