ironshell 1.5.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ironshell/__init__.py +682 -0
- ironshell/cloud.py +967 -0
- ironshell/distributed.py +691 -0
- ironshell/langchain.py +1074 -0
- ironshell/metrics.py +603 -0
- ironshell/py.typed +0 -0
- ironshell/workflows.py +539 -0
- ironshell-1.5.1.dist-info/METADATA +349 -0
- ironshell-1.5.1.dist-info/RECORD +11 -0
- ironshell-1.5.1.dist-info/WHEEL +5 -0
- ironshell-1.5.1.dist-info/top_level.txt +1 -0
ironshell/metrics.py
ADDED
|
@@ -0,0 +1,603 @@
|
|
|
1
|
+
"""
|
|
2
|
+
IronShell Metrics and Observability for Python
|
|
3
|
+
|
|
4
|
+
Provides metrics collection, tracing, and monitoring:
|
|
5
|
+
- Prometheus-compatible metrics
|
|
6
|
+
- OpenTelemetry integration
|
|
7
|
+
- Health checks
|
|
8
|
+
- Performance tracking
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import asyncio
|
|
14
|
+
import json
|
|
15
|
+
import time
|
|
16
|
+
from collections import defaultdict
|
|
17
|
+
from dataclasses import dataclass, field
|
|
18
|
+
from datetime import datetime, timedelta
|
|
19
|
+
from enum import Enum
|
|
20
|
+
from typing import Any, Callable, Dict, List, Optional, TypeVar
|
|
21
|
+
from contextlib import contextmanager
|
|
22
|
+
import threading
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
"MetricsCollector",
|
|
26
|
+
"Counter",
|
|
27
|
+
"Gauge",
|
|
28
|
+
"Histogram",
|
|
29
|
+
"Timer",
|
|
30
|
+
"HealthCheck",
|
|
31
|
+
"HealthStatus",
|
|
32
|
+
"Span",
|
|
33
|
+
"Tracer",
|
|
34
|
+
"metrics_middleware",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
T = TypeVar("T")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class HealthStatus(Enum):
|
|
41
|
+
"""Health check status"""
|
|
42
|
+
HEALTHY = "healthy"
|
|
43
|
+
DEGRADED = "degraded"
|
|
44
|
+
UNHEALTHY = "unhealthy"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass
|
|
48
|
+
class Counter:
|
|
49
|
+
"""
|
|
50
|
+
Counter metric - monotonically increasing value.
|
|
51
|
+
|
|
52
|
+
Use for: request counts, error counts, completed tasks.
|
|
53
|
+
"""
|
|
54
|
+
name: str
|
|
55
|
+
description: str = ""
|
|
56
|
+
labels: Dict[str, str] = field(default_factory=dict)
|
|
57
|
+
_value: float = 0.0
|
|
58
|
+
_lock: threading.Lock = field(default_factory=threading.Lock)
|
|
59
|
+
|
|
60
|
+
def inc(self, amount: float = 1.0) -> None:
|
|
61
|
+
"""Increment the counter"""
|
|
62
|
+
with self._lock:
|
|
63
|
+
self._value += amount
|
|
64
|
+
|
|
65
|
+
@property
|
|
66
|
+
def value(self) -> float:
|
|
67
|
+
"""Get current value"""
|
|
68
|
+
return self._value
|
|
69
|
+
|
|
70
|
+
def to_prometheus(self) -> str:
|
|
71
|
+
"""Export in Prometheus format"""
|
|
72
|
+
labels_str = ",".join(f'{k}="{v}"' for k, v in self.labels.items())
|
|
73
|
+
label_part = f"{{{labels_str}}}" if labels_str else ""
|
|
74
|
+
return f"{self.name}{label_part} {self._value}"
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass
|
|
78
|
+
class Gauge:
|
|
79
|
+
"""
|
|
80
|
+
Gauge metric - value that can go up and down.
|
|
81
|
+
|
|
82
|
+
Use for: current connections, queue size, memory usage.
|
|
83
|
+
"""
|
|
84
|
+
name: str
|
|
85
|
+
description: str = ""
|
|
86
|
+
labels: Dict[str, str] = field(default_factory=dict)
|
|
87
|
+
_value: float = 0.0
|
|
88
|
+
_lock: threading.Lock = field(default_factory=threading.Lock)
|
|
89
|
+
|
|
90
|
+
def set(self, value: float) -> None:
|
|
91
|
+
"""Set the gauge value"""
|
|
92
|
+
with self._lock:
|
|
93
|
+
self._value = value
|
|
94
|
+
|
|
95
|
+
def inc(self, amount: float = 1.0) -> None:
|
|
96
|
+
"""Increment the gauge"""
|
|
97
|
+
with self._lock:
|
|
98
|
+
self._value += amount
|
|
99
|
+
|
|
100
|
+
def dec(self, amount: float = 1.0) -> None:
|
|
101
|
+
"""Decrement the gauge"""
|
|
102
|
+
with self._lock:
|
|
103
|
+
self._value -= amount
|
|
104
|
+
|
|
105
|
+
@property
|
|
106
|
+
def value(self) -> float:
|
|
107
|
+
"""Get current value"""
|
|
108
|
+
return self._value
|
|
109
|
+
|
|
110
|
+
def to_prometheus(self) -> str:
|
|
111
|
+
"""Export in Prometheus format"""
|
|
112
|
+
labels_str = ",".join(f'{k}="{v}"' for k, v in self.labels.items())
|
|
113
|
+
label_part = f"{{{labels_str}}}" if labels_str else ""
|
|
114
|
+
return f"{self.name}{label_part} {self._value}"
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@dataclass
|
|
118
|
+
class Histogram:
|
|
119
|
+
"""
|
|
120
|
+
Histogram metric - distribution of values.
|
|
121
|
+
|
|
122
|
+
Use for: request latency, response sizes.
|
|
123
|
+
"""
|
|
124
|
+
name: str
|
|
125
|
+
description: str = ""
|
|
126
|
+
buckets: List[float] = field(default_factory=lambda: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0])
|
|
127
|
+
labels: Dict[str, str] = field(default_factory=dict)
|
|
128
|
+
_values: List[float] = field(default_factory=list)
|
|
129
|
+
_bucket_counts: Dict[float, int] = field(default_factory=dict)
|
|
130
|
+
_sum: float = 0.0
|
|
131
|
+
_count: int = 0
|
|
132
|
+
_lock: threading.Lock = field(default_factory=threading.Lock)
|
|
133
|
+
|
|
134
|
+
def __post_init__(self):
|
|
135
|
+
# Initialize bucket counts
|
|
136
|
+
for bucket in self.buckets:
|
|
137
|
+
self._bucket_counts[bucket] = 0
|
|
138
|
+
self._bucket_counts[float('inf')] = 0
|
|
139
|
+
|
|
140
|
+
def observe(self, value: float) -> None:
|
|
141
|
+
"""Record an observation"""
|
|
142
|
+
with self._lock:
|
|
143
|
+
self._values.append(value)
|
|
144
|
+
self._sum += value
|
|
145
|
+
self._count += 1
|
|
146
|
+
|
|
147
|
+
# Update bucket counts
|
|
148
|
+
for bucket in self.buckets:
|
|
149
|
+
if value <= bucket:
|
|
150
|
+
self._bucket_counts[bucket] += 1
|
|
151
|
+
self._bucket_counts[float('inf')] += 1
|
|
152
|
+
|
|
153
|
+
@property
|
|
154
|
+
def count(self) -> int:
|
|
155
|
+
"""Get observation count"""
|
|
156
|
+
return self._count
|
|
157
|
+
|
|
158
|
+
@property
|
|
159
|
+
def sum(self) -> float:
|
|
160
|
+
"""Get sum of observations"""
|
|
161
|
+
return self._sum
|
|
162
|
+
|
|
163
|
+
@property
|
|
164
|
+
def mean(self) -> float:
|
|
165
|
+
"""Get mean value"""
|
|
166
|
+
return self._sum / self._count if self._count > 0 else 0.0
|
|
167
|
+
|
|
168
|
+
def percentile(self, p: float) -> float:
|
|
169
|
+
"""Get percentile value"""
|
|
170
|
+
if not self._values:
|
|
171
|
+
return 0.0
|
|
172
|
+
sorted_values = sorted(self._values)
|
|
173
|
+
idx = int(len(sorted_values) * p / 100)
|
|
174
|
+
return sorted_values[min(idx, len(sorted_values) - 1)]
|
|
175
|
+
|
|
176
|
+
def to_prometheus(self) -> str:
|
|
177
|
+
"""Export in Prometheus format"""
|
|
178
|
+
lines = []
|
|
179
|
+
labels_str = ",".join(f'{k}="{v}"' for k, v in self.labels.items())
|
|
180
|
+
base_labels = f",{labels_str}" if labels_str else ""
|
|
181
|
+
|
|
182
|
+
for bucket, count in sorted(self._bucket_counts.items()):
|
|
183
|
+
le = "+Inf" if bucket == float('inf') else str(bucket)
|
|
184
|
+
lines.append(f'{self.name}_bucket{{le="{le}"{base_labels}}} {count}')
|
|
185
|
+
|
|
186
|
+
lines.append(f"{self.name}_sum{{{labels_str}}} {self._sum}" if labels_str else f"{self.name}_sum {self._sum}")
|
|
187
|
+
lines.append(f"{self.name}_count{{{labels_str}}} {self._count}" if labels_str else f"{self.name}_count {self._count}")
|
|
188
|
+
|
|
189
|
+
return "\n".join(lines)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
class Timer:
|
|
193
|
+
"""
|
|
194
|
+
Context manager for timing code blocks.
|
|
195
|
+
|
|
196
|
+
Records duration to a histogram metric.
|
|
197
|
+
"""
|
|
198
|
+
|
|
199
|
+
def __init__(self, histogram: Histogram):
|
|
200
|
+
self.histogram = histogram
|
|
201
|
+
self._start: Optional[float] = None
|
|
202
|
+
|
|
203
|
+
def __enter__(self) -> Timer:
|
|
204
|
+
self._start = time.perf_counter()
|
|
205
|
+
return self
|
|
206
|
+
|
|
207
|
+
def __exit__(self, *args) -> None:
|
|
208
|
+
if self._start is not None:
|
|
209
|
+
duration = time.perf_counter() - self._start
|
|
210
|
+
self.histogram.observe(duration)
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
@dataclass
|
|
214
|
+
class Span:
|
|
215
|
+
"""
|
|
216
|
+
Tracing span for distributed tracing.
|
|
217
|
+
"""
|
|
218
|
+
name: str
|
|
219
|
+
trace_id: str
|
|
220
|
+
span_id: str
|
|
221
|
+
parent_id: Optional[str] = None
|
|
222
|
+
start_time: Optional[datetime] = None
|
|
223
|
+
end_time: Optional[datetime] = None
|
|
224
|
+
attributes: Dict[str, Any] = field(default_factory=dict)
|
|
225
|
+
events: List[Dict[str, Any]] = field(default_factory=list)
|
|
226
|
+
status: str = "OK"
|
|
227
|
+
|
|
228
|
+
def set_attribute(self, key: str, value: Any) -> None:
|
|
229
|
+
"""Set span attribute"""
|
|
230
|
+
self.attributes[key] = value
|
|
231
|
+
|
|
232
|
+
def add_event(self, name: str, attributes: Optional[Dict[str, Any]] = None) -> None:
|
|
233
|
+
"""Add event to span"""
|
|
234
|
+
self.events.append({
|
|
235
|
+
"name": name,
|
|
236
|
+
"timestamp": datetime.now().isoformat(),
|
|
237
|
+
"attributes": attributes or {},
|
|
238
|
+
})
|
|
239
|
+
|
|
240
|
+
def set_status(self, status: str, description: Optional[str] = None) -> None:
|
|
241
|
+
"""Set span status"""
|
|
242
|
+
self.status = status
|
|
243
|
+
if description:
|
|
244
|
+
self.attributes["status_description"] = description
|
|
245
|
+
|
|
246
|
+
@property
|
|
247
|
+
def duration_ms(self) -> float:
|
|
248
|
+
"""Get span duration in milliseconds"""
|
|
249
|
+
if self.start_time and self.end_time:
|
|
250
|
+
return (self.end_time - self.start_time).total_seconds() * 1000
|
|
251
|
+
return 0.0
|
|
252
|
+
|
|
253
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
254
|
+
"""Convert to dictionary"""
|
|
255
|
+
return {
|
|
256
|
+
"name": self.name,
|
|
257
|
+
"traceId": self.trace_id,
|
|
258
|
+
"spanId": self.span_id,
|
|
259
|
+
"parentId": self.parent_id,
|
|
260
|
+
"startTime": self.start_time.isoformat() if self.start_time else None,
|
|
261
|
+
"endTime": self.end_time.isoformat() if self.end_time else None,
|
|
262
|
+
"attributes": self.attributes,
|
|
263
|
+
"events": self.events,
|
|
264
|
+
"status": self.status,
|
|
265
|
+
"durationMs": self.duration_ms,
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
class Tracer:
|
|
270
|
+
"""
|
|
271
|
+
Simple tracer for distributed tracing.
|
|
272
|
+
|
|
273
|
+
Creates and manages spans for request tracing.
|
|
274
|
+
"""
|
|
275
|
+
|
|
276
|
+
def __init__(self, service_name: str):
|
|
277
|
+
self.service_name = service_name
|
|
278
|
+
self._spans: List[Span] = []
|
|
279
|
+
self._current_span: Optional[Span] = None
|
|
280
|
+
self._lock = threading.Lock()
|
|
281
|
+
|
|
282
|
+
def _generate_id(self) -> str:
|
|
283
|
+
"""Generate unique ID"""
|
|
284
|
+
import random
|
|
285
|
+
return f"{random.randint(0, 0xFFFFFFFFFFFFFFFF):016x}"
|
|
286
|
+
|
|
287
|
+
@contextmanager
|
|
288
|
+
def start_span(
|
|
289
|
+
self,
|
|
290
|
+
name: str,
|
|
291
|
+
parent: Optional[Span] = None,
|
|
292
|
+
attributes: Optional[Dict[str, Any]] = None,
|
|
293
|
+
):
|
|
294
|
+
"""
|
|
295
|
+
Start a new span.
|
|
296
|
+
|
|
297
|
+
Args:
|
|
298
|
+
name: Span name
|
|
299
|
+
parent: Parent span (optional)
|
|
300
|
+
attributes: Initial attributes
|
|
301
|
+
|
|
302
|
+
Yields:
|
|
303
|
+
Span instance
|
|
304
|
+
"""
|
|
305
|
+
trace_id = parent.trace_id if parent else self._generate_id()
|
|
306
|
+
span = Span(
|
|
307
|
+
name=name,
|
|
308
|
+
trace_id=trace_id,
|
|
309
|
+
span_id=self._generate_id(),
|
|
310
|
+
parent_id=parent.span_id if parent else None,
|
|
311
|
+
start_time=datetime.now(),
|
|
312
|
+
attributes={"service.name": self.service_name, **(attributes or {})},
|
|
313
|
+
)
|
|
314
|
+
|
|
315
|
+
prev_span = self._current_span
|
|
316
|
+
self._current_span = span
|
|
317
|
+
|
|
318
|
+
try:
|
|
319
|
+
yield span
|
|
320
|
+
span.set_status("OK")
|
|
321
|
+
except Exception as e:
|
|
322
|
+
span.set_status("ERROR", str(e))
|
|
323
|
+
raise
|
|
324
|
+
finally:
|
|
325
|
+
span.end_time = datetime.now()
|
|
326
|
+
with self._lock:
|
|
327
|
+
self._spans.append(span)
|
|
328
|
+
self._current_span = prev_span
|
|
329
|
+
|
|
330
|
+
@property
|
|
331
|
+
def current_span(self) -> Optional[Span]:
|
|
332
|
+
"""Get current active span"""
|
|
333
|
+
return self._current_span
|
|
334
|
+
|
|
335
|
+
def get_spans(self) -> List[Span]:
|
|
336
|
+
"""Get all recorded spans"""
|
|
337
|
+
return self._spans.copy()
|
|
338
|
+
|
|
339
|
+
def clear_spans(self) -> None:
|
|
340
|
+
"""Clear recorded spans"""
|
|
341
|
+
with self._lock:
|
|
342
|
+
self._spans.clear()
|
|
343
|
+
|
|
344
|
+
def export_json(self) -> str:
|
|
345
|
+
"""Export spans as JSON"""
|
|
346
|
+
return json.dumps([s.to_dict() for s in self._spans], indent=2)
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
@dataclass
|
|
350
|
+
class HealthCheck:
|
|
351
|
+
"""
|
|
352
|
+
Health check for service monitoring.
|
|
353
|
+
"""
|
|
354
|
+
name: str
|
|
355
|
+
check_fn: Callable[[], bool]
|
|
356
|
+
timeout_ms: int = 5000
|
|
357
|
+
interval_ms: int = 30000
|
|
358
|
+
_last_check: Optional[datetime] = None
|
|
359
|
+
_last_status: HealthStatus = HealthStatus.HEALTHY
|
|
360
|
+
_consecutive_failures: int = 0
|
|
361
|
+
|
|
362
|
+
async def check(self) -> HealthStatus:
|
|
363
|
+
"""Run health check"""
|
|
364
|
+
try:
|
|
365
|
+
# Run check with timeout
|
|
366
|
+
if asyncio.iscoroutinefunction(self.check_fn):
|
|
367
|
+
result = await asyncio.wait_for(
|
|
368
|
+
self.check_fn(),
|
|
369
|
+
timeout=self.timeout_ms / 1000,
|
|
370
|
+
)
|
|
371
|
+
else:
|
|
372
|
+
result = await asyncio.wait_for(
|
|
373
|
+
asyncio.to_thread(self.check_fn),
|
|
374
|
+
timeout=self.timeout_ms / 1000,
|
|
375
|
+
)
|
|
376
|
+
|
|
377
|
+
if result:
|
|
378
|
+
self._consecutive_failures = 0
|
|
379
|
+
self._last_status = HealthStatus.HEALTHY
|
|
380
|
+
else:
|
|
381
|
+
self._consecutive_failures += 1
|
|
382
|
+
self._last_status = HealthStatus.DEGRADED if self._consecutive_failures < 3 else HealthStatus.UNHEALTHY
|
|
383
|
+
except asyncio.TimeoutError:
|
|
384
|
+
self._consecutive_failures += 1
|
|
385
|
+
self._last_status = HealthStatus.UNHEALTHY
|
|
386
|
+
except Exception:
|
|
387
|
+
self._consecutive_failures += 1
|
|
388
|
+
self._last_status = HealthStatus.UNHEALTHY
|
|
389
|
+
|
|
390
|
+
self._last_check = datetime.now()
|
|
391
|
+
return self._last_status
|
|
392
|
+
|
|
393
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
394
|
+
"""Convert to dictionary"""
|
|
395
|
+
return {
|
|
396
|
+
"name": self.name,
|
|
397
|
+
"status": self._last_status.value,
|
|
398
|
+
"lastCheck": self._last_check.isoformat() if self._last_check else None,
|
|
399
|
+
"consecutiveFailures": self._consecutive_failures,
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
class MetricsCollector:
|
|
404
|
+
"""
|
|
405
|
+
Central metrics collector.
|
|
406
|
+
|
|
407
|
+
Aggregates all metrics and provides export functionality.
|
|
408
|
+
"""
|
|
409
|
+
|
|
410
|
+
def __init__(self, namespace: str = "ironshell"):
|
|
411
|
+
self.namespace = namespace
|
|
412
|
+
self._counters: Dict[str, Counter] = {}
|
|
413
|
+
self._gauges: Dict[str, Gauge] = {}
|
|
414
|
+
self._histograms: Dict[str, Histogram] = {}
|
|
415
|
+
self._health_checks: Dict[str, HealthCheck] = {}
|
|
416
|
+
self._tracer: Optional[Tracer] = None
|
|
417
|
+
self._lock = threading.Lock()
|
|
418
|
+
|
|
419
|
+
def counter(
|
|
420
|
+
self,
|
|
421
|
+
name: str,
|
|
422
|
+
description: str = "",
|
|
423
|
+
labels: Optional[Dict[str, str]] = None,
|
|
424
|
+
) -> Counter:
|
|
425
|
+
"""Get or create a counter"""
|
|
426
|
+
full_name = f"{self.namespace}_{name}"
|
|
427
|
+
with self._lock:
|
|
428
|
+
if full_name not in self._counters:
|
|
429
|
+
self._counters[full_name] = Counter(
|
|
430
|
+
name=full_name,
|
|
431
|
+
description=description,
|
|
432
|
+
labels=labels or {},
|
|
433
|
+
)
|
|
434
|
+
return self._counters[full_name]
|
|
435
|
+
|
|
436
|
+
def gauge(
|
|
437
|
+
self,
|
|
438
|
+
name: str,
|
|
439
|
+
description: str = "",
|
|
440
|
+
labels: Optional[Dict[str, str]] = None,
|
|
441
|
+
) -> Gauge:
|
|
442
|
+
"""Get or create a gauge"""
|
|
443
|
+
full_name = f"{self.namespace}_{name}"
|
|
444
|
+
with self._lock:
|
|
445
|
+
if full_name not in self._gauges:
|
|
446
|
+
self._gauges[full_name] = Gauge(
|
|
447
|
+
name=full_name,
|
|
448
|
+
description=description,
|
|
449
|
+
labels=labels or {},
|
|
450
|
+
)
|
|
451
|
+
return self._gauges[full_name]
|
|
452
|
+
|
|
453
|
+
def histogram(
|
|
454
|
+
self,
|
|
455
|
+
name: str,
|
|
456
|
+
description: str = "",
|
|
457
|
+
buckets: Optional[List[float]] = None,
|
|
458
|
+
labels: Optional[Dict[str, str]] = None,
|
|
459
|
+
) -> Histogram:
|
|
460
|
+
"""Get or create a histogram"""
|
|
461
|
+
full_name = f"{self.namespace}_{name}"
|
|
462
|
+
with self._lock:
|
|
463
|
+
if full_name not in self._histograms:
|
|
464
|
+
self._histograms[full_name] = Histogram(
|
|
465
|
+
name=full_name,
|
|
466
|
+
description=description,
|
|
467
|
+
buckets=buckets or [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0],
|
|
468
|
+
labels=labels or {},
|
|
469
|
+
)
|
|
470
|
+
return self._histograms[full_name]
|
|
471
|
+
|
|
472
|
+
def register_health_check(
|
|
473
|
+
self,
|
|
474
|
+
name: str,
|
|
475
|
+
check_fn: Callable[[], bool],
|
|
476
|
+
timeout_ms: int = 5000,
|
|
477
|
+
) -> HealthCheck:
|
|
478
|
+
"""Register a health check"""
|
|
479
|
+
with self._lock:
|
|
480
|
+
self._health_checks[name] = HealthCheck(
|
|
481
|
+
name=name,
|
|
482
|
+
check_fn=check_fn,
|
|
483
|
+
timeout_ms=timeout_ms,
|
|
484
|
+
)
|
|
485
|
+
return self._health_checks[name]
|
|
486
|
+
|
|
487
|
+
def tracer(self, service_name: Optional[str] = None) -> Tracer:
|
|
488
|
+
"""Get or create tracer"""
|
|
489
|
+
if self._tracer is None:
|
|
490
|
+
self._tracer = Tracer(service_name or self.namespace)
|
|
491
|
+
return self._tracer
|
|
492
|
+
|
|
493
|
+
async def run_health_checks(self) -> Dict[str, HealthStatus]:
|
|
494
|
+
"""Run all health checks"""
|
|
495
|
+
results = {}
|
|
496
|
+
for name, check in self._health_checks.items():
|
|
497
|
+
results[name] = await check.check()
|
|
498
|
+
return results
|
|
499
|
+
|
|
500
|
+
def to_prometheus(self) -> str:
|
|
501
|
+
"""Export all metrics in Prometheus format"""
|
|
502
|
+
lines = []
|
|
503
|
+
|
|
504
|
+
for counter in self._counters.values():
|
|
505
|
+
if counter.description:
|
|
506
|
+
lines.append(f"# HELP {counter.name} {counter.description}")
|
|
507
|
+
lines.append(f"# TYPE {counter.name} counter")
|
|
508
|
+
lines.append(counter.to_prometheus())
|
|
509
|
+
|
|
510
|
+
for gauge in self._gauges.values():
|
|
511
|
+
if gauge.description:
|
|
512
|
+
lines.append(f"# HELP {gauge.name} {gauge.description}")
|
|
513
|
+
lines.append(f"# TYPE {gauge.name} gauge")
|
|
514
|
+
lines.append(gauge.to_prometheus())
|
|
515
|
+
|
|
516
|
+
for histogram in self._histograms.values():
|
|
517
|
+
if histogram.description:
|
|
518
|
+
lines.append(f"# HELP {histogram.name} {histogram.description}")
|
|
519
|
+
lines.append(f"# TYPE {histogram.name} histogram")
|
|
520
|
+
lines.append(histogram.to_prometheus())
|
|
521
|
+
|
|
522
|
+
return "\n".join(lines)
|
|
523
|
+
|
|
524
|
+
def to_json(self) -> str:
|
|
525
|
+
"""Export all metrics as JSON"""
|
|
526
|
+
data = {
|
|
527
|
+
"counters": {k: v.value for k, v in self._counters.items()},
|
|
528
|
+
"gauges": {k: v.value for k, v in self._gauges.items()},
|
|
529
|
+
"histograms": {
|
|
530
|
+
k: {
|
|
531
|
+
"count": v.count,
|
|
532
|
+
"sum": v.sum,
|
|
533
|
+
"mean": v.mean,
|
|
534
|
+
"p50": v.percentile(50),
|
|
535
|
+
"p95": v.percentile(95),
|
|
536
|
+
"p99": v.percentile(99),
|
|
537
|
+
}
|
|
538
|
+
for k, v in self._histograms.items()
|
|
539
|
+
},
|
|
540
|
+
"health_checks": {k: v.to_dict() for k, v in self._health_checks.items()},
|
|
541
|
+
}
|
|
542
|
+
return json.dumps(data, indent=2)
|
|
543
|
+
|
|
544
|
+
def reset(self) -> None:
|
|
545
|
+
"""Reset all metrics"""
|
|
546
|
+
with self._lock:
|
|
547
|
+
self._counters.clear()
|
|
548
|
+
self._gauges.clear()
|
|
549
|
+
self._histograms.clear()
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
# Singleton metrics collector
|
|
553
|
+
_default_collector: Optional[MetricsCollector] = None
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def get_metrics_collector() -> MetricsCollector:
|
|
557
|
+
"""Get the default metrics collector"""
|
|
558
|
+
global _default_collector
|
|
559
|
+
if _default_collector is None:
|
|
560
|
+
_default_collector = MetricsCollector()
|
|
561
|
+
return _default_collector
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def metrics_middleware(collector: Optional[MetricsCollector] = None):
|
|
565
|
+
"""
|
|
566
|
+
Decorator for adding metrics to functions.
|
|
567
|
+
|
|
568
|
+
Tracks call count, errors, and latency.
|
|
569
|
+
"""
|
|
570
|
+
mc = collector or get_metrics_collector()
|
|
571
|
+
|
|
572
|
+
def decorator(func: Callable[..., T]) -> Callable[..., T]:
|
|
573
|
+
name = func.__name__
|
|
574
|
+
calls = mc.counter(f"{name}_calls_total", "Total function calls")
|
|
575
|
+
errors = mc.counter(f"{name}_errors_total", "Total function errors")
|
|
576
|
+
latency = mc.histogram(f"{name}_latency_seconds", "Function latency")
|
|
577
|
+
|
|
578
|
+
if asyncio.iscoroutinefunction(func):
|
|
579
|
+
async def async_wrapper(*args, **kwargs) -> T:
|
|
580
|
+
calls.inc()
|
|
581
|
+
start = time.perf_counter()
|
|
582
|
+
try:
|
|
583
|
+
return await func(*args, **kwargs)
|
|
584
|
+
except Exception as e:
|
|
585
|
+
errors.inc()
|
|
586
|
+
raise
|
|
587
|
+
finally:
|
|
588
|
+
latency.observe(time.perf_counter() - start)
|
|
589
|
+
return async_wrapper
|
|
590
|
+
else:
|
|
591
|
+
def sync_wrapper(*args, **kwargs) -> T:
|
|
592
|
+
calls.inc()
|
|
593
|
+
start = time.perf_counter()
|
|
594
|
+
try:
|
|
595
|
+
return func(*args, **kwargs)
|
|
596
|
+
except Exception as e:
|
|
597
|
+
errors.inc()
|
|
598
|
+
raise
|
|
599
|
+
finally:
|
|
600
|
+
latency.observe(time.perf_counter() - start)
|
|
601
|
+
return sync_wrapper
|
|
602
|
+
|
|
603
|
+
return decorator
|
ironshell/py.typed
ADDED
|
File without changes
|