ko-observability 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ko_observability/__init__.py +15 -0
- ko_observability/client.py +67 -0
- ko_observability/exporters.py +208 -0
- ko_observability/otel.py +262 -0
- ko_observability/otlp_config.py +88 -0
- ko_observability/py.typed +0 -0
- ko_observability/query.py +436 -0
- ko_observability/structlog_bridge.py +115 -0
- ko_observability-0.2.0.dist-info/METADATA +110 -0
- ko_observability-0.2.0.dist-info/RECORD +11 -0
- ko_observability-0.2.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Nexora org-wide OpenTelemetry libraries."""
|
|
2
|
+
|
|
3
|
+
from ko_observability.client import get_meter, get_tracer
|
|
4
|
+
from ko_observability.otel import init_otel
|
|
5
|
+
from ko_observability.structlog_bridge import make_otel_bridge, otel_bridge
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"init_otel",
|
|
9
|
+
"get_tracer",
|
|
10
|
+
"get_meter",
|
|
11
|
+
"otel_bridge",
|
|
12
|
+
"make_otel_bridge",
|
|
13
|
+
]
|
|
14
|
+
|
|
15
|
+
__version__ = "0.2.0"
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Lazy OTel client helpers for library and application code.
|
|
2
|
+
|
|
3
|
+
Use these when you want a Tracer/Meter without requiring the host process to
|
|
4
|
+
have called ``init_otel`` first (e.g. inside a reusable library). Each
|
|
5
|
+
function bootstraps a minimal provider on first use if -- and only if -- no
|
|
6
|
+
provider has been registered yet by the host process.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import os
|
|
12
|
+
import threading
|
|
13
|
+
|
|
14
|
+
from opentelemetry import metrics as _otel_metrics
|
|
15
|
+
from opentelemetry import trace
|
|
16
|
+
|
|
17
|
+
from ko_observability.exporters import make_span_exporter
|
|
18
|
+
from ko_observability.otlp_config import sdk_disabled
|
|
19
|
+
|
|
20
|
+
_init_lock = threading.Lock()
|
|
21
|
+
_initialized = False
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def get_tracer(name: str, version: str | None = None):
|
|
25
|
+
"""Return a Tracer, lazily bootstrapping a minimal provider if needed."""
|
|
26
|
+
_ensure_trace_provider()
|
|
27
|
+
return trace.get_tracer(name, version)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def get_meter(name: str, version: str | None = None) -> _otel_metrics.Meter:
|
|
31
|
+
"""Return a Meter (no-op if metrics provider is not configured)."""
|
|
32
|
+
return _otel_metrics.get_meter(name, version or "")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _ensure_trace_provider() -> None:
|
|
36
|
+
global _initialized
|
|
37
|
+
if _initialized:
|
|
38
|
+
return
|
|
39
|
+
with _init_lock:
|
|
40
|
+
if _initialized:
|
|
41
|
+
return
|
|
42
|
+
if sdk_disabled():
|
|
43
|
+
_initialized = True
|
|
44
|
+
return
|
|
45
|
+
|
|
46
|
+
provider = trace.get_tracer_provider()
|
|
47
|
+
if "NoOp" not in type(provider).__name__ and "Proxy" not in type(provider).__name__:
|
|
48
|
+
_initialized = True
|
|
49
|
+
return
|
|
50
|
+
|
|
51
|
+
try:
|
|
52
|
+
from opentelemetry.sdk.resources import Resource
|
|
53
|
+
from opentelemetry.sdk.trace import TracerProvider
|
|
54
|
+
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
55
|
+
|
|
56
|
+
resource = Resource.create(
|
|
57
|
+
{
|
|
58
|
+
"service.name": os.getenv("OTEL_SERVICE_NAME", "nexora-service"),
|
|
59
|
+
"service.namespace": os.getenv("OTEL_SERVICE_NAMESPACE", "nexora"),
|
|
60
|
+
}
|
|
61
|
+
)
|
|
62
|
+
tp = TracerProvider(resource=resource)
|
|
63
|
+
tp.add_span_processor(BatchSpanProcessor(make_span_exporter()))
|
|
64
|
+
trace.set_tracer_provider(tp)
|
|
65
|
+
except Exception:
|
|
66
|
+
pass
|
|
67
|
+
_initialized = True
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
"""Exporter factories: local JSONL files (default) or OTLP/gRPC, per TELEMETRY_DESTINATION.
|
|
2
|
+
|
|
3
|
+
File exporters write one OTLP/JSON ``Export*ServiceRequest`` per line -- the
|
|
4
|
+
same encoding the Collector's ``otlpjsonfile`` receiver replays -- with
|
|
5
|
+
``traceId``/``spanId``/``parentSpanId`` rewritten from protobuf-JSON base64
|
|
6
|
+
to the spec's hex form. Append-only, one ``write()`` per batch, mutex-guarded.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import base64
|
|
12
|
+
import binascii
|
|
13
|
+
import json
|
|
14
|
+
import logging
|
|
15
|
+
import os
|
|
16
|
+
import threading
|
|
17
|
+
from collections.abc import Sequence
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from opentelemetry.sdk._logs.export import LogRecordExporter, LogRecordExportResult
|
|
21
|
+
from opentelemetry.sdk.metrics.export import MetricExporter, MetricExportResult, MetricsData
|
|
22
|
+
from opentelemetry.sdk.trace import ReadableSpan
|
|
23
|
+
from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
|
|
24
|
+
|
|
25
|
+
from ko_observability.otlp_config import (
|
|
26
|
+
otlp_endpoint,
|
|
27
|
+
otlp_exporter_insecure,
|
|
28
|
+
telemetry_destination,
|
|
29
|
+
telemetry_file_dir,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
_logger = logging.getLogger(__name__)
|
|
33
|
+
|
|
34
|
+
TRACES_FILE = "traces.jsonl"
|
|
35
|
+
METRICS_FILE = "metrics.jsonl"
|
|
36
|
+
LOGS_FILE = "logs.jsonl"
|
|
37
|
+
|
|
38
|
+
_ID_KEYS = frozenset({"traceId", "spanId", "parentSpanId"})
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _b64_to_hex(value: str) -> str:
|
|
42
|
+
try:
|
|
43
|
+
return base64.b64decode(value, validate=True).hex()
|
|
44
|
+
except (binascii.Error, ValueError):
|
|
45
|
+
return value
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _hex_ids(obj: Any) -> Any:
|
|
49
|
+
if isinstance(obj, dict):
|
|
50
|
+
return {
|
|
51
|
+
k: (_b64_to_hex(v) if k in _ID_KEYS and isinstance(v, str) else _hex_ids(v))
|
|
52
|
+
for k, v in obj.items()
|
|
53
|
+
}
|
|
54
|
+
if isinstance(obj, list):
|
|
55
|
+
return [_hex_ids(v) for v in obj]
|
|
56
|
+
return obj
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _to_otlp_json(message: Any) -> dict[str, Any]:
|
|
60
|
+
from google.protobuf.json_format import MessageToDict
|
|
61
|
+
|
|
62
|
+
return _hex_ids(MessageToDict(message, use_integers_for_enums=True))
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class _JsonlWriter:
|
|
66
|
+
def __init__(self, path: str) -> None:
|
|
67
|
+
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
|
|
68
|
+
self._fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o644)
|
|
69
|
+
self._lock = threading.Lock()
|
|
70
|
+
|
|
71
|
+
def write_line(self, obj: dict[str, Any]) -> None:
|
|
72
|
+
data = (json.dumps(obj, separators=(",", ":")) + "\n").encode("utf-8")
|
|
73
|
+
with self._lock:
|
|
74
|
+
if self._fd < 0:
|
|
75
|
+
raise OSError("telemetry file closed")
|
|
76
|
+
view = memoryview(data)
|
|
77
|
+
written = 0
|
|
78
|
+
try:
|
|
79
|
+
while view:
|
|
80
|
+
n = os.write(self._fd, view)
|
|
81
|
+
written += n
|
|
82
|
+
view = view[n:]
|
|
83
|
+
except OSError:
|
|
84
|
+
# A partial write before the failure leaves a fragment (no
|
|
85
|
+
# trailing newline) on disk -- but only if this call made any
|
|
86
|
+
# progress at all; a failure on the very first byte leaves the
|
|
87
|
+
# file exactly as the last successful write_line left it (already
|
|
88
|
+
# newline-terminated), so no recovery byte is needed there.
|
|
89
|
+
# Left alone, the next successful export would append onto the
|
|
90
|
+
# fragment and produce one unparseable JSONL line, corrupting
|
|
91
|
+
# previously-good data rather than just the failed batch.
|
|
92
|
+
# Best-effort: terminate the fragment with a newline so line
|
|
93
|
+
# framing survives, then propagate the original error (never
|
|
94
|
+
# crash the host).
|
|
95
|
+
if written > 0:
|
|
96
|
+
try:
|
|
97
|
+
os.write(self._fd, b"\n")
|
|
98
|
+
except OSError:
|
|
99
|
+
pass
|
|
100
|
+
raise
|
|
101
|
+
|
|
102
|
+
def close(self) -> None:
|
|
103
|
+
with self._lock:
|
|
104
|
+
if self._fd >= 0:
|
|
105
|
+
os.close(self._fd)
|
|
106
|
+
self._fd = -1
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _default_path(name: str) -> str:
|
|
110
|
+
return os.path.join(telemetry_file_dir(), name)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class FileSpanExporter(SpanExporter):
|
|
114
|
+
def __init__(self, path: str | None = None) -> None:
|
|
115
|
+
self._writer = _JsonlWriter(path or _default_path(TRACES_FILE))
|
|
116
|
+
|
|
117
|
+
def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
|
|
118
|
+
try:
|
|
119
|
+
from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans
|
|
120
|
+
|
|
121
|
+
self._writer.write_line(_to_otlp_json(encode_spans(spans)))
|
|
122
|
+
return SpanExportResult.SUCCESS
|
|
123
|
+
except Exception as exc:
|
|
124
|
+
_logger.warning("telemetry.file_export_failed: traces: %s", exc)
|
|
125
|
+
return SpanExportResult.FAILURE
|
|
126
|
+
|
|
127
|
+
def shutdown(self) -> None:
|
|
128
|
+
self._writer.close()
|
|
129
|
+
|
|
130
|
+
def force_flush(self, timeout_millis: int = 30000) -> bool:
|
|
131
|
+
return True
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class FileLogExporter(LogRecordExporter):
|
|
135
|
+
def __init__(self, path: str | None = None) -> None:
|
|
136
|
+
self._writer = _JsonlWriter(path or _default_path(LOGS_FILE))
|
|
137
|
+
|
|
138
|
+
def export(self, batch: Sequence[Any]) -> LogRecordExportResult:
|
|
139
|
+
try:
|
|
140
|
+
from opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs
|
|
141
|
+
|
|
142
|
+
self._writer.write_line(_to_otlp_json(encode_logs(batch)))
|
|
143
|
+
return LogRecordExportResult.SUCCESS
|
|
144
|
+
except Exception as exc:
|
|
145
|
+
_logger.warning("telemetry.file_export_failed: logs: %s", exc)
|
|
146
|
+
return LogRecordExportResult.FAILURE
|
|
147
|
+
|
|
148
|
+
def shutdown(self) -> None:
|
|
149
|
+
self._writer.close()
|
|
150
|
+
|
|
151
|
+
def force_flush(self, timeout_millis: int = 30000) -> bool:
|
|
152
|
+
return True
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
class FileMetricExporter(MetricExporter):
|
|
156
|
+
def __init__(
|
|
157
|
+
self,
|
|
158
|
+
path: str | None = None,
|
|
159
|
+
preferred_temporality: dict[type, Any] | None = None,
|
|
160
|
+
preferred_aggregation: dict[type, Any] | None = None,
|
|
161
|
+
) -> None:
|
|
162
|
+
super().__init__(
|
|
163
|
+
preferred_temporality=preferred_temporality,
|
|
164
|
+
preferred_aggregation=preferred_aggregation,
|
|
165
|
+
)
|
|
166
|
+
self._writer = _JsonlWriter(path or _default_path(METRICS_FILE))
|
|
167
|
+
|
|
168
|
+
def export(
|
|
169
|
+
self, metrics_data: MetricsData, timeout_millis: float = 10_000, **kwargs: Any
|
|
170
|
+
) -> MetricExportResult:
|
|
171
|
+
try:
|
|
172
|
+
from opentelemetry.exporter.otlp.proto.common.metrics_encoder import encode_metrics
|
|
173
|
+
|
|
174
|
+
self._writer.write_line(_to_otlp_json(encode_metrics(metrics_data)))
|
|
175
|
+
return MetricExportResult.SUCCESS
|
|
176
|
+
except Exception as exc:
|
|
177
|
+
_logger.warning("telemetry.file_export_failed: metrics: %s", exc)
|
|
178
|
+
return MetricExportResult.FAILURE
|
|
179
|
+
|
|
180
|
+
def force_flush(self, timeout_millis: float = 10_000) -> bool:
|
|
181
|
+
return True
|
|
182
|
+
|
|
183
|
+
def shutdown(self, timeout_millis: float = 30_000, **kwargs: Any) -> None:
|
|
184
|
+
self._writer.close()
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def make_span_exporter() -> SpanExporter:
|
|
188
|
+
if telemetry_destination() == "file":
|
|
189
|
+
return FileSpanExporter()
|
|
190
|
+
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
|
|
191
|
+
|
|
192
|
+
return OTLPSpanExporter(endpoint=otlp_endpoint(), insecure=otlp_exporter_insecure())
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def make_log_exporter() -> LogRecordExporter:
|
|
196
|
+
if telemetry_destination() == "file":
|
|
197
|
+
return FileLogExporter()
|
|
198
|
+
from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
|
|
199
|
+
|
|
200
|
+
return OTLPLogExporter(endpoint=otlp_endpoint(), insecure=otlp_exporter_insecure())
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def make_metric_exporter() -> MetricExporter:
|
|
204
|
+
if telemetry_destination() == "file":
|
|
205
|
+
return FileMetricExporter()
|
|
206
|
+
from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
|
|
207
|
+
|
|
208
|
+
return OTLPMetricExporter(endpoint=otlp_endpoint(), insecure=otlp_exporter_insecure())
|
ko_observability/otel.py
ADDED
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
"""OpenTelemetry bootstrap for Nexora services.
|
|
2
|
+
|
|
3
|
+
Single public entry point: ``init_otel``. Reads standard OTEL_* env vars,
|
|
4
|
+
builds a Resource, configures exporters (local JSONL files by default,
|
|
5
|
+
OTLP/gRPC when TELEMETRY_DESTINATION=otlp), and registers global providers.
|
|
6
|
+
Optionally attaches FastAPI middleware. Idempotent. No-ops if
|
|
7
|
+
OTEL_SDK_DISABLED=true.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import logging
|
|
13
|
+
import math
|
|
14
|
+
import os
|
|
15
|
+
import threading
|
|
16
|
+
from collections.abc import Sequence
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
from opentelemetry.context import Context
|
|
20
|
+
from opentelemetry.sdk.trace import Span, SpanProcessor
|
|
21
|
+
|
|
22
|
+
from ko_observability.exporters import (
|
|
23
|
+
make_log_exporter,
|
|
24
|
+
make_metric_exporter,
|
|
25
|
+
make_span_exporter,
|
|
26
|
+
)
|
|
27
|
+
from ko_observability.otlp_config import sdk_disabled
|
|
28
|
+
|
|
29
|
+
_logger = logging.getLogger(__name__)
|
|
30
|
+
|
|
31
|
+
_init_lock = threading.Lock()
|
|
32
|
+
_initialized: bool = False
|
|
33
|
+
_current_service_name: str | None = None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def get_current_service_name() -> str | None:
|
|
37
|
+
"""Return the ``service_name`` passed to ``init_otel``, or ``None`` if
|
|
38
|
+
it hasn't been called yet in this process. Used by ``query.py`` to
|
|
39
|
+
self-scope read access to the calling service's own telemetry without
|
|
40
|
+
requiring the caller to repeat the service name."""
|
|
41
|
+
return _current_service_name
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class StructlogContextSpanProcessor(SpanProcessor):
|
|
45
|
+
"""Stamp span attributes from structlog contextvars.
|
|
46
|
+
|
|
47
|
+
``attr_map`` is a sequence of ``(contextvar_key, span_attribute_key)``
|
|
48
|
+
pairs copied from ``structlog.contextvars.get_contextvars()`` onto every
|
|
49
|
+
new span. Empty by default -- pass your app's own correlation keys, e.g.
|
|
50
|
+
``(("request_id", "http.request_id"),)``.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
def __init__(self, attr_map: Sequence[tuple[str, str]] = ()) -> None:
|
|
54
|
+
self._attr_map = tuple(attr_map)
|
|
55
|
+
|
|
56
|
+
def on_start(self, span: Span, parent_context: Context | None = None) -> None:
|
|
57
|
+
if not self._attr_map:
|
|
58
|
+
return
|
|
59
|
+
try:
|
|
60
|
+
from structlog.contextvars import get_contextvars
|
|
61
|
+
|
|
62
|
+
ctx = get_contextvars()
|
|
63
|
+
for ctx_key, attr_key in self._attr_map:
|
|
64
|
+
if ctx_key not in ctx:
|
|
65
|
+
continue
|
|
66
|
+
value = ctx[ctx_key]
|
|
67
|
+
if value is not None:
|
|
68
|
+
span.set_attribute(attr_key, str(value))
|
|
69
|
+
except Exception:
|
|
70
|
+
pass
|
|
71
|
+
|
|
72
|
+
def on_end(self, span) -> None:
|
|
73
|
+
return
|
|
74
|
+
|
|
75
|
+
def shutdown(self) -> None:
|
|
76
|
+
return
|
|
77
|
+
|
|
78
|
+
def force_flush(self, timeout_millis: int = 30000) -> bool:
|
|
79
|
+
return True
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def init_otel(
|
|
83
|
+
service_name: str,
|
|
84
|
+
service_namespace: str = "nexora",
|
|
85
|
+
instrument_fastapi_app: Any | None = None,
|
|
86
|
+
enable_logs: bool = True,
|
|
87
|
+
enable_metrics: bool = True,
|
|
88
|
+
extra_metric_readers: Sequence[Any] | None = None,
|
|
89
|
+
span_processors: Sequence[SpanProcessor] | None = None,
|
|
90
|
+
context_attr_map: Sequence[tuple[str, str]] = (),
|
|
91
|
+
) -> None:
|
|
92
|
+
"""Initialize OpenTelemetry for this process.
|
|
93
|
+
|
|
94
|
+
Args:
|
|
95
|
+
service_name: Resource ``service.name``.
|
|
96
|
+
service_namespace: Resource ``service.namespace`` (shared org label).
|
|
97
|
+
instrument_fastapi_app: Optional FastAPI app for HTTP auto-instrumentation.
|
|
98
|
+
enable_logs: Register OTLP log export when LOG_DESTINATION is otlp/both.
|
|
99
|
+
enable_metrics: Register OTLP metric export.
|
|
100
|
+
extra_metric_readers: Additional metric readers (e.g. in-memory admin reader).
|
|
101
|
+
span_processors: Processors registered before the batch exporter (after
|
|
102
|
+
the default StructlogContextSpanProcessor).
|
|
103
|
+
context_attr_map: ``(contextvar_key, span_attribute_key)`` pairs stamped
|
|
104
|
+
onto every span from structlog contextvars, e.g.
|
|
105
|
+
``(("request_id", "http.request_id"),)``. Empty by default.
|
|
106
|
+
"""
|
|
107
|
+
global _initialized, _current_service_name
|
|
108
|
+
with _init_lock:
|
|
109
|
+
if _initialized:
|
|
110
|
+
if instrument_fastapi_app is not None:
|
|
111
|
+
_try_instrument_fastapi(instrument_fastapi_app)
|
|
112
|
+
return
|
|
113
|
+
|
|
114
|
+
# Recorded on this first (and only effective) call, even down the
|
|
115
|
+
# disabled/exception paths below -- get_current_service_name()
|
|
116
|
+
# reflects "what service_name was this process configured with",
|
|
117
|
+
# independent of whether export actually succeeded.
|
|
118
|
+
_current_service_name = service_name
|
|
119
|
+
|
|
120
|
+
if sdk_disabled():
|
|
121
|
+
from opentelemetry import trace
|
|
122
|
+
from opentelemetry.trace import NoOpTracerProvider
|
|
123
|
+
|
|
124
|
+
trace.set_tracer_provider(NoOpTracerProvider())
|
|
125
|
+
_initialized = True
|
|
126
|
+
return
|
|
127
|
+
|
|
128
|
+
try:
|
|
129
|
+
from opentelemetry import trace
|
|
130
|
+
from opentelemetry.sdk.resources import Resource
|
|
131
|
+
from opentelemetry.sdk.trace import TracerProvider
|
|
132
|
+
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
133
|
+
|
|
134
|
+
resource_attrs = {
|
|
135
|
+
"service.name": service_name,
|
|
136
|
+
"service.namespace": service_namespace,
|
|
137
|
+
"service.version": os.getenv("SERVICE_VERSION", "0.0.0"),
|
|
138
|
+
"deployment.environment": os.getenv("DEPLOYMENT_ENV", "development"),
|
|
139
|
+
}
|
|
140
|
+
resource = Resource.create(resource_attrs)
|
|
141
|
+
provider = TracerProvider(resource=resource)
|
|
142
|
+
|
|
143
|
+
provider.add_span_processor(StructlogContextSpanProcessor(context_attr_map))
|
|
144
|
+
if span_processors:
|
|
145
|
+
for proc in span_processors:
|
|
146
|
+
provider.add_span_processor(proc)
|
|
147
|
+
|
|
148
|
+
provider.add_span_processor(BatchSpanProcessor(make_span_exporter()))
|
|
149
|
+
|
|
150
|
+
trace.set_tracer_provider(provider)
|
|
151
|
+
|
|
152
|
+
log_destination = os.getenv("LOG_DESTINATION", "stdout").strip().lower()
|
|
153
|
+
if enable_logs and log_destination in ("otlp", "both"):
|
|
154
|
+
_init_logs(resource)
|
|
155
|
+
|
|
156
|
+
if enable_metrics:
|
|
157
|
+
_init_metrics(resource, extra_metric_readers)
|
|
158
|
+
|
|
159
|
+
_try_instrument_httpx()
|
|
160
|
+
_try_instrument_elasticsearch()
|
|
161
|
+
if instrument_fastapi_app is not None:
|
|
162
|
+
_try_instrument_fastapi(instrument_fastapi_app)
|
|
163
|
+
|
|
164
|
+
_initialized = True
|
|
165
|
+
except Exception as exc:
|
|
166
|
+
_logger.warning("otel.init_failed: %s", exc)
|
|
167
|
+
_initialized = True
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _init_logs(resource: Any) -> None:
|
|
171
|
+
from opentelemetry._logs import set_logger_provider
|
|
172
|
+
from opentelemetry.sdk._logs import LoggerProvider
|
|
173
|
+
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
|
174
|
+
|
|
175
|
+
log_provider = LoggerProvider(resource=resource)
|
|
176
|
+
log_provider.add_log_record_processor(BatchLogRecordProcessor(make_log_exporter()))
|
|
177
|
+
set_logger_provider(log_provider)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _metric_export_interval_ms() -> int:
|
|
181
|
+
"""Parse ``OTEL_METRIC_EXPORT_INTERVAL``, falling back to 60000 ms.
|
|
182
|
+
|
|
183
|
+
The env var is used iff it parses as a finite number greater than zero;
|
|
184
|
+
anything else (unset, non-numeric, negative, zero, NaN/Infinity) silently
|
|
185
|
+
falls back to the default. Matches the JS/Go SDKs' behavior.
|
|
186
|
+
"""
|
|
187
|
+
raw = os.getenv("OTEL_METRIC_EXPORT_INTERVAL", "")
|
|
188
|
+
try:
|
|
189
|
+
value = float(raw)
|
|
190
|
+
except (TypeError, ValueError):
|
|
191
|
+
return 60000
|
|
192
|
+
if not math.isfinite(value) or value <= 0:
|
|
193
|
+
return 60000
|
|
194
|
+
return int(value)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _init_metrics(resource: Any, extra_metric_readers: Sequence[Any] | None) -> None:
|
|
198
|
+
from opentelemetry import metrics as otel_metrics
|
|
199
|
+
from opentelemetry.sdk.metrics import MeterProvider
|
|
200
|
+
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
|
201
|
+
|
|
202
|
+
readers: list[Any] = [
|
|
203
|
+
PeriodicExportingMetricReader(
|
|
204
|
+
exporter=make_metric_exporter(),
|
|
205
|
+
export_interval_millis=_metric_export_interval_ms(),
|
|
206
|
+
)
|
|
207
|
+
]
|
|
208
|
+
if extra_metric_readers:
|
|
209
|
+
readers.extend(extra_metric_readers)
|
|
210
|
+
|
|
211
|
+
provider = MeterProvider(resource=resource, metric_readers=readers)
|
|
212
|
+
otel_metrics.set_meter_provider(provider)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _try_instrument_httpx() -> None:
|
|
216
|
+
try:
|
|
217
|
+
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
|
|
218
|
+
|
|
219
|
+
HTTPXClientInstrumentor().instrument()
|
|
220
|
+
except Exception:
|
|
221
|
+
pass
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _try_instrument_elasticsearch() -> None:
|
|
225
|
+
try:
|
|
226
|
+
from opentelemetry.instrumentation.elasticsearch import (
|
|
227
|
+
ElasticsearchInstrumentor,
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
ElasticsearchInstrumentor().instrument()
|
|
231
|
+
except Exception:
|
|
232
|
+
pass
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _fastapi_server_request_hook(span: Any, scope: dict) -> None:
|
|
236
|
+
"""Rename the span to ``METHOD /matched/route/template`` when a route
|
|
237
|
+
actually matched. Deliberately does NOT fall back to the raw request
|
|
238
|
+
path for unmatched routes (404s, health-check probes hitting unknown
|
|
239
|
+
paths, etc.) -- that would let unbounded, attacker-controlled path
|
|
240
|
+
segments become span names/cardinality. Unmatched requests keep
|
|
241
|
+
FastAPIInstrumentor's default span name."""
|
|
242
|
+
try:
|
|
243
|
+
if span is None or not span.is_recording():
|
|
244
|
+
return
|
|
245
|
+
route = scope.get("route", None)
|
|
246
|
+
path = getattr(route, "path", None)
|
|
247
|
+
if not path:
|
|
248
|
+
return
|
|
249
|
+
method = scope.get("method", "")
|
|
250
|
+
if method:
|
|
251
|
+
span.update_name(f"{method} {path}")
|
|
252
|
+
except Exception:
|
|
253
|
+
pass
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _try_instrument_fastapi(app: Any) -> None:
|
|
257
|
+
try:
|
|
258
|
+
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
|
|
259
|
+
|
|
260
|
+
FastAPIInstrumentor.instrument_app(app, server_request_hook=_fastapi_server_request_hook)
|
|
261
|
+
except Exception:
|
|
262
|
+
pass
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""OTLP exporter settings shared by trace, log, and metric exporters."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import os
|
|
7
|
+
|
|
8
|
+
_logger = logging.getLogger(__name__)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def parse_bool_env(raw: str) -> bool | None:
|
|
12
|
+
"""Boolean-ish env parsing shared across the Python/JS/Go SDKs.
|
|
13
|
+
|
|
14
|
+
Trims whitespace, lowercases, then compares. Truthy: ``true``, ``1``,
|
|
15
|
+
``yes``. Falsy: ``false``, ``0``, ``no``. Anything else (including an
|
|
16
|
+
empty/unset value) is "not specified" -- returns ``None`` so the caller
|
|
17
|
+
can apply its own default.
|
|
18
|
+
"""
|
|
19
|
+
value = raw.strip().lower()
|
|
20
|
+
if value in ("true", "1", "yes"):
|
|
21
|
+
return True
|
|
22
|
+
if value in ("false", "0", "no"):
|
|
23
|
+
return False
|
|
24
|
+
return None
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def sdk_disabled() -> bool:
|
|
28
|
+
"""Whether ``OTEL_SDK_DISABLED`` is truthy (see ``parse_bool_env``)."""
|
|
29
|
+
return parse_bool_env(os.getenv("OTEL_SDK_DISABLED", "")) is True
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def otlp_endpoint() -> str:
|
|
33
|
+
"""OTLP/gRPC endpoint (standard ``OTEL_EXPORTER_OTLP_ENDPOINT``).
|
|
34
|
+
|
|
35
|
+
Set-but-empty or whitespace-only (e.g. a compose file's
|
|
36
|
+
``OTEL_EXPORTER_OTLP_ENDPOINT=`` line) is treated the same as unset,
|
|
37
|
+
matching the JS (``||``) and Go (``if v != ""``) SDKs.
|
|
38
|
+
"""
|
|
39
|
+
return os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "").strip() or "http://localhost:4317"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def otlp_exporter_insecure() -> bool:
|
|
43
|
+
"""Whether OTLP gRPC exporters should use an insecure channel.
|
|
44
|
+
|
|
45
|
+
Honors ``OTEL_EXPORTER_OTLP_INSECURE`` when set (explicit truthy ->
|
|
46
|
+
plaintext, explicit falsy -> TLS, even when the endpoint is
|
|
47
|
+
``http://`` -- passing a concrete bool to the exporter constructors
|
|
48
|
+
overrides their own scheme inference). Otherwise infers from the
|
|
49
|
+
endpoint scheme: ``http://`` -> insecure, ``https://`` -> TLS.
|
|
50
|
+
"""
|
|
51
|
+
flag = parse_bool_env(os.getenv("OTEL_EXPORTER_OTLP_INSECURE", ""))
|
|
52
|
+
if flag is not None:
|
|
53
|
+
return flag
|
|
54
|
+
return otlp_endpoint().startswith("http://")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
_TELEMETRY_DESTINATIONS = ("file", "otlp")
|
|
58
|
+
_warned_invalid_destination = False
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def telemetry_destination() -> str:
|
|
62
|
+
"""Where exporters send: ``"file"`` (default) or ``"otlp"``.
|
|
63
|
+
|
|
64
|
+
``TELEMETRY_DESTINATION`` is trimmed and lowercased. Unset/empty -> ``file``.
|
|
65
|
+
Anything unrecognized -> ``file`` plus one ``telemetry.destination_invalid``
|
|
66
|
+
warning per process. Identical parsing in the JS/Go SDKs.
|
|
67
|
+
"""
|
|
68
|
+
global _warned_invalid_destination
|
|
69
|
+
raw = os.getenv("TELEMETRY_DESTINATION", "")
|
|
70
|
+
value = raw.strip().lower()
|
|
71
|
+
if value == "":
|
|
72
|
+
return "file"
|
|
73
|
+
if value in _TELEMETRY_DESTINATIONS:
|
|
74
|
+
return value
|
|
75
|
+
if not _warned_invalid_destination:
|
|
76
|
+
_warned_invalid_destination = True
|
|
77
|
+
_logger.warning("telemetry.destination_invalid: %r (using 'file')", raw)
|
|
78
|
+
return "file"
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def telemetry_file_dir() -> str:
|
|
82
|
+
"""Directory for ``traces.jsonl``/``metrics.jsonl``/``logs.jsonl`` in file mode.
|
|
83
|
+
|
|
84
|
+
``TELEMETRY_FILE_DIR`` trimmed; unset/blank -> ``./telemetry`` (relative to
|
|
85
|
+
the process CWD; no ``~`` expansion, matching JS/Go).
|
|
86
|
+
"""
|
|
87
|
+
value = os.getenv("TELEMETRY_FILE_DIR", "").strip()
|
|
88
|
+
return value or "./telemetry"
|
|
File without changes
|
|
@@ -0,0 +1,436 @@
|
|
|
1
|
+
"""Read-only query helpers for admin-panel style access to logs/traces/
|
|
2
|
+
metrics this service already emitted via ``init_otel``. Optional -- requires
|
|
3
|
+
the ``query`` extra (``pip install ko-observability[query]``); nothing
|
|
4
|
+
else in this package imports this module, so producer services that only
|
|
5
|
+
emit telemetry never pull in its dependency (``httpx``).
|
|
6
|
+
|
|
7
|
+
Deliberately **not** backend-agnostic. OpenTelemetry's own ecosystem
|
|
8
|
+
convention keeps write (SDK) and read (backend-native tooling: Kibana,
|
|
9
|
+
Grafana, a vendor's own API) separate on purpose -- different backends have
|
|
10
|
+
fundamentally different query shapes, and a genuine abstraction over
|
|
11
|
+
"any backend" ends up either thin-and-leaky or a full reimplementation of
|
|
12
|
+
what those tools already do. This is a thin, Elasticsearch-specific helper
|
|
13
|
+
matching the ``mapping.mode: otel`` field layout the gateway Collector
|
|
14
|
+
writes to (see ``docs/otel.md``'s Architecture section). If a second
|
|
15
|
+
backend is ever actually added, build its own client rather than forcing
|
|
16
|
+
one interface over both.
|
|
17
|
+
|
|
18
|
+
Security model: every method defaults to scoping results to the *calling
|
|
19
|
+
process's own* ``service_name`` (read from ``init_otel``'s
|
|
20
|
+
``get_current_service_name()``, or the ``OTEL_SERVICE_NAME`` env var as a
|
|
21
|
+
fallback) -- so one app's admin panel can't casually see another app's
|
|
22
|
+
telemetry through this helper alone. This is a **library-layer**
|
|
23
|
+
convenience, not a security boundary: the Elasticsearch credentials this
|
|
24
|
+
reads with should themselves be scoped server-side (a per-service API key
|
|
25
|
+
with document-level security restricting to that service's
|
|
26
|
+
``resource.attributes.service.name``) before exposing this to anything less
|
|
27
|
+
trusted than the app's own operators. See ``docs/query.md``.
|
|
28
|
+
|
|
29
|
+
Field-path note: ES field mappings here (``@timestamp``, ``trace_id``,
|
|
30
|
+
``span_id``, ``parent_span_id``, ``severity_text``, ``body.text``,
|
|
31
|
+
``status.code``, ``resource.attributes.*``, ``attributes.*``,
|
|
32
|
+
``metrics.<dotted.name>``) are verified against a live cluster by the e2e
|
|
33
|
+
suite (``e2e/test_query.py``, ``e2e/test_traces.py``, ``e2e/test_metrics.py``):
|
|
34
|
+
all filter fields are keyword (exact ``term``), ``status.code`` is OTel
|
|
35
|
+
title-case (``"Error"``, not ``"ERROR"``), ``body.text`` is analyzed
|
|
36
|
+
(``match``), and ``metrics.<dotted.name>`` is indexed as a literal dotted
|
|
37
|
+
key -- ``_extract_metric_value``'s nested-object branch is a defensive
|
|
38
|
+
fallback for an alternate mapping shape, not the one actually observed.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
from __future__ import annotations
|
|
42
|
+
|
|
43
|
+
import os
|
|
44
|
+
from dataclasses import dataclass, field
|
|
45
|
+
from typing import Any
|
|
46
|
+
from urllib.parse import urlsplit, urlunsplit
|
|
47
|
+
|
|
48
|
+
import httpx
|
|
49
|
+
|
|
50
|
+
from ko_observability.otel import get_current_service_name
|
|
51
|
+
|
|
52
|
+
TRACES_INDEX = "traces-generic.otel-default"
|
|
53
|
+
LOGS_INDEX = "logs-generic.otel-default"
|
|
54
|
+
METRICS_INDEX = "metrics-generic.otel-default"
|
|
55
|
+
|
|
56
|
+
_DEFAULT_ES_URL = "https://localhost:9200"
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass
|
|
60
|
+
class LogEntry:
|
|
61
|
+
timestamp: str
|
|
62
|
+
severity: str | None
|
|
63
|
+
body: str
|
|
64
|
+
trace_id: str | None
|
|
65
|
+
span_id: str | None
|
|
66
|
+
attributes: dict[str, Any] = field(default_factory=dict)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass
|
|
70
|
+
class TraceSummary:
|
|
71
|
+
trace_id: str
|
|
72
|
+
span_id: str
|
|
73
|
+
name: str
|
|
74
|
+
duration_ms: float | None
|
|
75
|
+
status: str | None
|
|
76
|
+
start_time: str
|
|
77
|
+
attributes: dict[str, Any] = field(default_factory=dict)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@dataclass
|
|
81
|
+
class Span:
|
|
82
|
+
span_id: str
|
|
83
|
+
parent_span_id: str | None
|
|
84
|
+
trace_id: str
|
|
85
|
+
name: str
|
|
86
|
+
duration_ms: float | None
|
|
87
|
+
status: str | None
|
|
88
|
+
start_time: str
|
|
89
|
+
attributes: dict[str, Any] = field(default_factory=dict)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@dataclass
|
|
93
|
+
class MetricPoint:
|
|
94
|
+
timestamp: str
|
|
95
|
+
value: float
|
|
96
|
+
attributes: dict[str, Any] = field(default_factory=dict)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@dataclass
|
|
100
|
+
class ExecutionResult:
|
|
101
|
+
"""Combined result for ``get_execution`` -- everything correlated to one
|
|
102
|
+
custom attribute value (e.g. a task/job execution id) across both the
|
|
103
|
+
traces and logs indices."""
|
|
104
|
+
|
|
105
|
+
execution_id: str
|
|
106
|
+
traces: list[TraceSummary]
|
|
107
|
+
logs: list[LogEntry]
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class QueryClientError(RuntimeError):
|
|
111
|
+
"""Raised for ES connection/auth failures or missing service_name."""
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _resolve_since(since: str) -> str:
|
|
115
|
+
"""Accepts either an ES date-math expression (``"now-1h"``, passed
|
|
116
|
+
through unchanged) or a bare duration shorthand (``"1h"``, ``"15m"``,
|
|
117
|
+
``"2d"``) which gets ``now-`` prepended."""
|
|
118
|
+
return since if since.startswith("now") else f"now-{since}"
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _attribute_filters(attributes: dict[str, str] | None) -> list[dict[str, Any]]:
|
|
122
|
+
if not attributes:
|
|
123
|
+
return []
|
|
124
|
+
return [{"term": {f"attributes.{key}": value}} for key, value in attributes.items()]
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _redact_url(url: str) -> str:
|
|
128
|
+
"""Strip any userinfo (``user:pass@``) from a URL before it's allowed
|
|
129
|
+
into an exception message. ``ELASTICSEARCH_URL``/``es_url`` may carry
|
|
130
|
+
embedded basic-auth credentials (httpx picks those up automatically for
|
|
131
|
+
the actual request) -- those must never leak into a raised error."""
|
|
132
|
+
parts = urlsplit(url)
|
|
133
|
+
if "@" not in (parts.netloc or ""):
|
|
134
|
+
return url
|
|
135
|
+
netloc = parts.netloc.rsplit("@", 1)[-1]
|
|
136
|
+
return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment))
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
class QueryClient:
|
|
140
|
+
"""Thin Elasticsearch query client scoped to one service's telemetry.
|
|
141
|
+
|
|
142
|
+
Prefer ``get_query_client()`` over constructing this directly -- it
|
|
143
|
+
resolves ``service_name`` and connection settings from the same
|
|
144
|
+
environment ``init_otel`` already uses.
|
|
145
|
+
"""
|
|
146
|
+
|
|
147
|
+
def __init__(
|
|
148
|
+
self,
|
|
149
|
+
service_name: str,
|
|
150
|
+
*,
|
|
151
|
+
es_url: str | None = None,
|
|
152
|
+
username: str | None = None,
|
|
153
|
+
password: str | None = None,
|
|
154
|
+
api_key: str | None = None,
|
|
155
|
+
ca_certs: str | None = None,
|
|
156
|
+
timeout: float = 10.0,
|
|
157
|
+
) -> None:
|
|
158
|
+
self.service_name = service_name
|
|
159
|
+
self._es_url = (es_url or os.getenv("ELASTICSEARCH_URL") or _DEFAULT_ES_URL).rstrip("/")
|
|
160
|
+
# Redacted once up front -- this is the only form of the ES URL that
|
|
161
|
+
# may ever reach an exception message (self._es_url itself may carry
|
|
162
|
+
# embedded basic-auth credentials; see _redact_url).
|
|
163
|
+
self._es_url_display = _redact_url(self._es_url)
|
|
164
|
+
|
|
165
|
+
headers: dict[str, str] = {}
|
|
166
|
+
auth: tuple[str, str] | None = None
|
|
167
|
+
resolved_api_key = api_key or os.getenv("ELASTICSEARCH_API_KEY")
|
|
168
|
+
if resolved_api_key:
|
|
169
|
+
headers["Authorization"] = f"ApiKey {resolved_api_key}"
|
|
170
|
+
else:
|
|
171
|
+
resolved_username = (
|
|
172
|
+
username if username is not None else os.getenv("ELASTICSEARCH_USERNAME", "elastic")
|
|
173
|
+
)
|
|
174
|
+
resolved_password = (
|
|
175
|
+
password
|
|
176
|
+
or os.getenv("ELASTICSEARCH_PASSWORD")
|
|
177
|
+
or os.getenv("ELASTIC_PASSWORD") # base-stack env var name
|
|
178
|
+
)
|
|
179
|
+
if resolved_password:
|
|
180
|
+
auth = (resolved_username, resolved_password)
|
|
181
|
+
|
|
182
|
+
verify: str | bool = ca_certs or os.getenv("ELASTICSEARCH_CA_CERTS") or True
|
|
183
|
+
|
|
184
|
+
self._client = httpx.Client(
|
|
185
|
+
base_url=self._es_url,
|
|
186
|
+
headers=headers,
|
|
187
|
+
auth=auth,
|
|
188
|
+
verify=verify,
|
|
189
|
+
timeout=timeout,
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
def close(self) -> None:
|
|
193
|
+
self._client.close()
|
|
194
|
+
|
|
195
|
+
def __enter__(self) -> QueryClient:
|
|
196
|
+
return self
|
|
197
|
+
|
|
198
|
+
def __exit__(self, *exc_info: object) -> None:
|
|
199
|
+
self.close()
|
|
200
|
+
|
|
201
|
+
def _search(
|
|
202
|
+
self,
|
|
203
|
+
index: str,
|
|
204
|
+
query: dict[str, Any],
|
|
205
|
+
*,
|
|
206
|
+
sort: list[dict[str, Any]],
|
|
207
|
+
size: int,
|
|
208
|
+
) -> list[dict[str, Any]]:
|
|
209
|
+
try:
|
|
210
|
+
response = self._client.post(
|
|
211
|
+
f"/{index}/_search",
|
|
212
|
+
json={"query": query, "sort": sort, "size": size},
|
|
213
|
+
)
|
|
214
|
+
except httpx.HTTPError as exc:
|
|
215
|
+
raise QueryClientError(
|
|
216
|
+
f"failed to reach Elasticsearch at {self._es_url_display}: {exc}"
|
|
217
|
+
) from exc
|
|
218
|
+
|
|
219
|
+
if response.status_code == 404:
|
|
220
|
+
# Data stream not created yet (no documents written) -- not an error.
|
|
221
|
+
return []
|
|
222
|
+
if response.is_error:
|
|
223
|
+
raise QueryClientError(
|
|
224
|
+
f"Elasticsearch query against '{index}' failed "
|
|
225
|
+
f"({response.status_code}): {response.text}"
|
|
226
|
+
)
|
|
227
|
+
return [hit["_source"] for hit in response.json()["hits"]["hits"]]
|
|
228
|
+
|
|
229
|
+
def list_logs(
|
|
230
|
+
self,
|
|
231
|
+
since: str = "1h",
|
|
232
|
+
*,
|
|
233
|
+
level: str | None = None,
|
|
234
|
+
search_text: str | None = None,
|
|
235
|
+
attributes: dict[str, str] | None = None,
|
|
236
|
+
limit: int = 100,
|
|
237
|
+
) -> list[LogEntry]:
|
|
238
|
+
filters: list[dict[str, Any]] = [
|
|
239
|
+
{"term": {"resource.attributes.service.name": self.service_name}},
|
|
240
|
+
{"range": {"@timestamp": {"gte": _resolve_since(since)}}},
|
|
241
|
+
*_attribute_filters(attributes),
|
|
242
|
+
]
|
|
243
|
+
if level:
|
|
244
|
+
# Under the OTel index mapping severity_text is a keyword; a term
|
|
245
|
+
# query hits it directly. OTel severity_text is already upper-case
|
|
246
|
+
# (INFO/WARN/ERROR/...).
|
|
247
|
+
filters.append({"term": {"severity_text": level.upper()}})
|
|
248
|
+
|
|
249
|
+
must: list[dict[str, Any]] = []
|
|
250
|
+
if search_text:
|
|
251
|
+
must.append({"match": {"body.text": search_text}})
|
|
252
|
+
|
|
253
|
+
docs = self._search(
|
|
254
|
+
LOGS_INDEX,
|
|
255
|
+
{"bool": {"filter": filters, "must": must}},
|
|
256
|
+
sort=[{"@timestamp": "desc"}],
|
|
257
|
+
size=limit,
|
|
258
|
+
)
|
|
259
|
+
return [
|
|
260
|
+
LogEntry(
|
|
261
|
+
timestamp=doc.get("@timestamp", ""),
|
|
262
|
+
severity=doc.get("severity_text"),
|
|
263
|
+
body=doc.get("body", {}).get("text", ""),
|
|
264
|
+
trace_id=doc.get("trace_id"),
|
|
265
|
+
span_id=doc.get("span_id"),
|
|
266
|
+
attributes=doc.get("attributes", {}),
|
|
267
|
+
)
|
|
268
|
+
for doc in docs
|
|
269
|
+
]
|
|
270
|
+
|
|
271
|
+
def list_traces(
|
|
272
|
+
self,
|
|
273
|
+
since: str = "1h",
|
|
274
|
+
*,
|
|
275
|
+
status: str | None = None,
|
|
276
|
+
attributes: dict[str, str] | None = None,
|
|
277
|
+
limit: int = 50,
|
|
278
|
+
) -> list[TraceSummary]:
|
|
279
|
+
filters: list[dict[str, Any]] = [
|
|
280
|
+
{"term": {"resource.attributes.service.name": self.service_name}},
|
|
281
|
+
{"range": {"@timestamp": {"gte": _resolve_since(since)}}},
|
|
282
|
+
*_attribute_filters(attributes),
|
|
283
|
+
]
|
|
284
|
+
must_not: list[dict[str, Any]] = [{"exists": {"field": "parent_span_id"}}]
|
|
285
|
+
if status:
|
|
286
|
+
# Under the OTel index mapping status.code is a keyword, hit
|
|
287
|
+
# directly by term. Its serialized value is OTel title-case
|
|
288
|
+
# ("Error"/"Ok"/"Unset"), NOT upper-case. str.capitalize() maps
|
|
289
|
+
# error->Error, ok->Ok, unset->Unset.
|
|
290
|
+
filters.append({"term": {"status.code": status.capitalize()}})
|
|
291
|
+
|
|
292
|
+
docs = self._search(
|
|
293
|
+
TRACES_INDEX,
|
|
294
|
+
{"bool": {"filter": filters, "must_not": must_not}},
|
|
295
|
+
sort=[{"@timestamp": "desc"}],
|
|
296
|
+
size=limit,
|
|
297
|
+
)
|
|
298
|
+
return [_trace_summary_from_doc(doc) for doc in docs]
|
|
299
|
+
|
|
300
|
+
def get_trace(self, trace_id: str, *, limit: int = 1000) -> list[Span]:
|
|
301
|
+
"""All spans for ``trace_id``, scoped -- like every other method on
|
|
302
|
+
this client -- to this client's own ``service_name``. Consequence:
|
|
303
|
+
spans contributed to the same distributed trace by *other* services
|
|
304
|
+
are not returned by this call; querying the full cross-service trace
|
|
305
|
+
requires calling ``get_trace`` against each contributing service's
|
|
306
|
+
own QueryClient.
|
|
307
|
+
"""
|
|
308
|
+
docs = self._search(
|
|
309
|
+
TRACES_INDEX,
|
|
310
|
+
{
|
|
311
|
+
"bool": {
|
|
312
|
+
"filter": [
|
|
313
|
+
{"term": {"trace_id": trace_id}},
|
|
314
|
+
{"term": {"resource.attributes.service.name": self.service_name}},
|
|
315
|
+
]
|
|
316
|
+
}
|
|
317
|
+
},
|
|
318
|
+
sort=[{"@timestamp": "asc"}],
|
|
319
|
+
size=limit,
|
|
320
|
+
)
|
|
321
|
+
return [
|
|
322
|
+
Span(
|
|
323
|
+
span_id=doc.get("span_id", ""),
|
|
324
|
+
parent_span_id=doc.get("parent_span_id"),
|
|
325
|
+
trace_id=doc.get("trace_id", trace_id),
|
|
326
|
+
name=doc.get("name", ""),
|
|
327
|
+
duration_ms=_duration_ms(doc),
|
|
328
|
+
status=doc.get("status", {}).get("code") if doc.get("status") else None,
|
|
329
|
+
start_time=doc.get("@timestamp", ""),
|
|
330
|
+
attributes=doc.get("attributes", {}),
|
|
331
|
+
)
|
|
332
|
+
for doc in docs
|
|
333
|
+
]
|
|
334
|
+
|
|
335
|
+
def get_metric(
|
|
336
|
+
self, metric_name: str, since: str = "1h", *, limit: int = 500
|
|
337
|
+
) -> list[MetricPoint]:
|
|
338
|
+
filters = [
|
|
339
|
+
{"term": {"resource.attributes.service.name": self.service_name}},
|
|
340
|
+
{"range": {"@timestamp": {"gte": _resolve_since(since)}}},
|
|
341
|
+
{"exists": {"field": f"metrics.{metric_name}"}},
|
|
342
|
+
]
|
|
343
|
+
docs = self._search(
|
|
344
|
+
METRICS_INDEX,
|
|
345
|
+
{"bool": {"filter": filters}},
|
|
346
|
+
sort=[{"@timestamp": "asc"}],
|
|
347
|
+
size=limit,
|
|
348
|
+
)
|
|
349
|
+
points = []
|
|
350
|
+
for doc in docs:
|
|
351
|
+
value = _extract_metric_value(doc.get("metrics", {}), metric_name)
|
|
352
|
+
if value is None:
|
|
353
|
+
continue
|
|
354
|
+
points.append(
|
|
355
|
+
MetricPoint(
|
|
356
|
+
timestamp=doc.get("@timestamp", ""),
|
|
357
|
+
value=value,
|
|
358
|
+
attributes=doc.get("attributes", {}),
|
|
359
|
+
)
|
|
360
|
+
)
|
|
361
|
+
return points
|
|
362
|
+
|
|
363
|
+
def get_execution(
|
|
364
|
+
self,
|
|
365
|
+
execution_id: str,
|
|
366
|
+
*,
|
|
367
|
+
attr_key: str = "execution_id",
|
|
368
|
+
since: str = "7d",
|
|
369
|
+
limit: int = 200,
|
|
370
|
+
) -> ExecutionResult:
|
|
371
|
+
"""Everything correlated to one custom attribute value (a task/job
|
|
372
|
+
execution id, a request id, whatever an app chose to stamp) --
|
|
373
|
+
across both traces and logs, in one call. Requires the app to have
|
|
374
|
+
stamped that attribute itself: pass it via ``init_otel``'s
|
|
375
|
+
``context_attr_map`` (for spans) and ``make_otel_bridge``'s
|
|
376
|
+
``correlation_keys`` (for logs) at startup -- see
|
|
377
|
+
``docs/onboarding-a-new-module.md``.
|
|
378
|
+
"""
|
|
379
|
+
attrs = {attr_key: execution_id}
|
|
380
|
+
return ExecutionResult(
|
|
381
|
+
execution_id=execution_id,
|
|
382
|
+
traces=self.list_traces(since=since, attributes=attrs, limit=limit),
|
|
383
|
+
logs=self.list_logs(since=since, attributes=attrs, limit=limit),
|
|
384
|
+
)
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def _trace_summary_from_doc(doc: dict[str, Any]) -> TraceSummary:
|
|
388
|
+
status = doc.get("status", {})
|
|
389
|
+
return TraceSummary(
|
|
390
|
+
trace_id=doc.get("trace_id", ""),
|
|
391
|
+
span_id=doc.get("span_id", ""),
|
|
392
|
+
name=doc.get("name", ""),
|
|
393
|
+
duration_ms=_duration_ms(doc),
|
|
394
|
+
status=status.get("code") if status else None,
|
|
395
|
+
start_time=doc.get("@timestamp", ""),
|
|
396
|
+
attributes=doc.get("attributes", {}),
|
|
397
|
+
)
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def _duration_ms(doc: dict[str, Any]) -> float | None:
|
|
401
|
+
# `duration` in the ES otel mapping is nanoseconds; e2e/test_traces.py
|
|
402
|
+
# confirms the field exists but not its unit -- adjust the divisor here
|
|
403
|
+
# if a live run shows otherwise.
|
|
404
|
+
duration_ns = doc.get("duration")
|
|
405
|
+
return duration_ns / 1_000_000 if duration_ns is not None else None
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def _extract_metric_value(metrics_obj: dict[str, Any], dotted_name: str) -> float | None:
|
|
409
|
+
"""`metrics.<name>` may be indexed as a literal dotted key or as nested
|
|
410
|
+
objects matching the name's dot segments, depending on the ES mapping.
|
|
411
|
+
Try both."""
|
|
412
|
+
if dotted_name in metrics_obj:
|
|
413
|
+
value = metrics_obj[dotted_name]
|
|
414
|
+
return value if isinstance(value, (int, float)) else None
|
|
415
|
+
node: Any = metrics_obj
|
|
416
|
+
for part in dotted_name.split("."):
|
|
417
|
+
if not isinstance(node, dict) or part not in node:
|
|
418
|
+
return None
|
|
419
|
+
node = node[part]
|
|
420
|
+
return node if isinstance(node, (int, float)) else None
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def get_query_client(service_name: str | None = None, **kwargs: Any) -> QueryClient:
|
|
424
|
+
"""Return a QueryClient scoped to ``service_name``, falling back to the
|
|
425
|
+
current process's own service (from ``init_otel``, or the
|
|
426
|
+
``OTEL_SERVICE_NAME`` env var). Raises QueryClientError if neither is
|
|
427
|
+
available -- pass ``service_name`` explicitly or call ``init_otel``
|
|
428
|
+
first.
|
|
429
|
+
"""
|
|
430
|
+
resolved = service_name or get_current_service_name() or os.getenv("OTEL_SERVICE_NAME")
|
|
431
|
+
if not resolved:
|
|
432
|
+
raise QueryClientError(
|
|
433
|
+
"no service_name available: pass one explicitly, call init_otel() first, "
|
|
434
|
+
"or set OTEL_SERVICE_NAME"
|
|
435
|
+
)
|
|
436
|
+
return QueryClient(resolved, **kwargs)
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""Structlog -> OpenTelemetry log bridge."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from opentelemetry import trace
|
|
9
|
+
from opentelemetry._logs import LogRecord, SeverityNumber, get_logger_provider
|
|
10
|
+
|
|
11
|
+
_LEVEL_TO_SEVERITY: dict[str, SeverityNumber] = {
|
|
12
|
+
"debug": SeverityNumber.DEBUG,
|
|
13
|
+
"info": SeverityNumber.INFO,
|
|
14
|
+
"warning": SeverityNumber.WARN,
|
|
15
|
+
"warn": SeverityNumber.WARN,
|
|
16
|
+
"error": SeverityNumber.ERROR,
|
|
17
|
+
"critical": SeverityNumber.FATAL,
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _get_provider_or_none():
|
|
22
|
+
"""Return the registered LoggerProvider, or ``None`` if none is configured.
|
|
23
|
+
|
|
24
|
+
``get_logger_provider()`` defaults to a ``ProxyLoggerProvider`` (not a
|
|
25
|
+
``NoOpLoggerProvider``) when nothing has been registered yet, so both must
|
|
26
|
+
be treated as "unconfigured" -- matches the class-name check in
|
|
27
|
+
``client.py``'s ``_ensure_trace_provider``.
|
|
28
|
+
"""
|
|
29
|
+
provider = get_logger_provider()
|
|
30
|
+
if provider is None or type(provider).__name__ in ("NoOpLoggerProvider", "ProxyLoggerProvider"):
|
|
31
|
+
return None
|
|
32
|
+
return provider
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def make_otel_bridge(
|
|
36
|
+
correlation_keys: frozenset[str] = frozenset(),
|
|
37
|
+
attribute_prefix: str = "",
|
|
38
|
+
logger_name: str = "ko_observability.structlog",
|
|
39
|
+
):
|
|
40
|
+
"""Build a structlog processor that emits one OTel LogRecord per event.
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
correlation_keys: event_dict keys namespaced under ``attribute_prefix``
|
|
44
|
+
on the emitted LogRecord (e.g. ``{"request_id", "run_id"}``).
|
|
45
|
+
Every other key is passed through unprefixed as a LogRecord
|
|
46
|
+
attribute -- except ``event``, ``level``, ``timestamp``, and
|
|
47
|
+
``@timestamp``, which are always dropped from ``attributes``
|
|
48
|
+
because they're already carried by the LogRecord's own
|
|
49
|
+
``body``/``severity_text``/``timestamp`` fields. Empty by
|
|
50
|
+
default.
|
|
51
|
+
attribute_prefix: prefix applied to ``correlation_keys`` attributes,
|
|
52
|
+
e.g. ``"myapp."``. Ignored if ``correlation_keys`` is empty.
|
|
53
|
+
logger_name: name registered with the OTel LoggerProvider.
|
|
54
|
+
|
|
55
|
+
Returns a structlog processor with signature
|
|
56
|
+
``(logger, method_name, event_dict) -> event_dict``. Never raises --
|
|
57
|
+
any failure building or emitting the LogRecord is swallowed and the
|
|
58
|
+
event_dict is passed through unchanged, so a bridge bug can't break the
|
|
59
|
+
app's logging.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
def _build_attributes(event_dict: dict[str, Any]) -> dict[str, Any]:
|
|
63
|
+
attributes: dict[str, Any] = {}
|
|
64
|
+
for key, value in event_dict.items():
|
|
65
|
+
if key in ("event", "level", "timestamp", "@timestamp"):
|
|
66
|
+
continue
|
|
67
|
+
if key in correlation_keys:
|
|
68
|
+
attributes[f"{attribute_prefix}{key}"] = value
|
|
69
|
+
else:
|
|
70
|
+
attributes[key] = value
|
|
71
|
+
return attributes
|
|
72
|
+
|
|
73
|
+
def _otel_bridge(_logger, _method_name, event_dict: dict[str, Any]) -> dict[str, Any]:
|
|
74
|
+
# Never break the app's logging: any failure here (a bad provider,
|
|
75
|
+
# an unexpected event_dict shape, an exporter-side error surfacing
|
|
76
|
+
# through emit()) is swallowed and the event still reaches
|
|
77
|
+
# structlog's other processors/renderer unchanged.
|
|
78
|
+
try:
|
|
79
|
+
provider = _get_provider_or_none()
|
|
80
|
+
if provider is None:
|
|
81
|
+
return event_dict
|
|
82
|
+
|
|
83
|
+
span = trace.get_current_span()
|
|
84
|
+
ctx = span.get_span_context() if span is not None else None
|
|
85
|
+
trace_id = ctx.trace_id if ctx and ctx.is_valid else 0
|
|
86
|
+
span_id = ctx.span_id if ctx and ctx.is_valid else 0
|
|
87
|
+
trace_flags = ctx.trace_flags if ctx and ctx.is_valid else None
|
|
88
|
+
|
|
89
|
+
level = str(event_dict.get("level", "info")).lower()
|
|
90
|
+
severity = _LEVEL_TO_SEVERITY.get(level, SeverityNumber.INFO)
|
|
91
|
+
body = event_dict.get("event", "")
|
|
92
|
+
|
|
93
|
+
now_ns = time.time_ns()
|
|
94
|
+
record = LogRecord(
|
|
95
|
+
timestamp=now_ns,
|
|
96
|
+
observed_timestamp=now_ns,
|
|
97
|
+
trace_id=trace_id,
|
|
98
|
+
span_id=span_id,
|
|
99
|
+
trace_flags=trace_flags,
|
|
100
|
+
severity_text=level.upper(),
|
|
101
|
+
severity_number=severity,
|
|
102
|
+
body=body,
|
|
103
|
+
attributes=_build_attributes(event_dict),
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
otel_logger = provider.get_logger(logger_name)
|
|
107
|
+
otel_logger.emit(record)
|
|
108
|
+
except Exception:
|
|
109
|
+
pass
|
|
110
|
+
return event_dict
|
|
111
|
+
|
|
112
|
+
return _otel_bridge
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
otel_bridge = make_otel_bridge()
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: ko-observability
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Org-wide OpenTelemetry bootstrap for Nexora Python services
|
|
5
|
+
Project-URL: Repository, https://github.com/Komus-ai/nexora-observability
|
|
6
|
+
Project-URL: Documentation, https://github.com/Komus-ai/nexora-observability/tree/main/docs
|
|
7
|
+
Author-email: Komus-ai <dev@cinco.ai>
|
|
8
|
+
License: MIT
|
|
9
|
+
Keywords: observability,opentelemetry,otlp,structlog
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Requires-Python: >=3.12
|
|
16
|
+
Requires-Dist: opentelemetry-api>=1.27.0
|
|
17
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-common>=1.27.0
|
|
18
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-grpc>=1.27.0
|
|
19
|
+
Requires-Dist: opentelemetry-sdk>=1.27.0
|
|
20
|
+
Requires-Dist: protobuf<8.0,>=5.0
|
|
21
|
+
Requires-Dist: structlog>=24.4.0
|
|
22
|
+
Provides-Extra: dev
|
|
23
|
+
Requires-Dist: httpx>=0.27.0; extra == 'dev'
|
|
24
|
+
Requires-Dist: mypy>=1.13.0; extra == 'dev'
|
|
25
|
+
Requires-Dist: opentelemetry-sdk>=1.27.0; extra == 'dev'
|
|
26
|
+
Requires-Dist: pytest>=8.3.0; extra == 'dev'
|
|
27
|
+
Requires-Dist: ruff>=0.8.0; extra == 'dev'
|
|
28
|
+
Provides-Extra: elasticsearch
|
|
29
|
+
Requires-Dist: opentelemetry-instrumentation-elasticsearch>=0.48b0; extra == 'elasticsearch'
|
|
30
|
+
Provides-Extra: fastapi
|
|
31
|
+
Requires-Dist: fastapi>=0.115.0; extra == 'fastapi'
|
|
32
|
+
Requires-Dist: opentelemetry-instrumentation-fastapi>=0.48b0; extra == 'fastapi'
|
|
33
|
+
Provides-Extra: httpx
|
|
34
|
+
Requires-Dist: opentelemetry-instrumentation-httpx>=0.48b0; extra == 'httpx'
|
|
35
|
+
Provides-Extra: query
|
|
36
|
+
Requires-Dist: httpx>=0.27.0; extra == 'query'
|
|
37
|
+
Description-Content-Type: text/markdown
|
|
38
|
+
|
|
39
|
+
# ko-observability (Python)
|
|
40
|
+
|
|
41
|
+
Org-wide OpenTelemetry bootstrap for Nexora Python services: one-call, idempotent
|
|
42
|
+
`init_otel` for process startup, lazy `get_tracer` / `get_meter` accessors for library code
|
|
43
|
+
that doesn't own process lifecycle, and a structlog processor (`otel_bridge` /
|
|
44
|
+
`make_otel_bridge`) that stamps each log line with the active span's `trace_id`/`span_id`.
|
|
45
|
+
|
|
46
|
+
## Install
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
pip install ko-observability
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Optional extras, install alongside the base package as needed:
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
# Auto-instrument FastAPI request/response spans
|
|
56
|
+
pip install "ko-observability[fastapi]"
|
|
57
|
+
|
|
58
|
+
# Auto-instrument outbound httpx calls
|
|
59
|
+
pip install "ko-observability[httpx]"
|
|
60
|
+
|
|
61
|
+
# Auto-instrument the Elasticsearch client
|
|
62
|
+
pip install "ko-observability[elasticsearch]"
|
|
63
|
+
|
|
64
|
+
# Read your own service's telemetry back out (admin-panel style queries)
|
|
65
|
+
pip install "ko-observability[query]"
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## Quick start
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
from ko_observability import init_otel, get_tracer
|
|
72
|
+
|
|
73
|
+
init_otel(service_name="my-service", service_namespace="nexora")
|
|
74
|
+
|
|
75
|
+
tracer = get_tracer(__name__)
|
|
76
|
+
with tracer.start_as_current_span("do-work"):
|
|
77
|
+
...
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Call `init_otel` exactly once at process startup. It's safe to call again (or to let library
|
|
81
|
+
code fall back to `get_tracer`/`get_meter` before it runs) — both paths are no-ops on top of an
|
|
82
|
+
already-configured provider. `OTEL_SDK_DISABLED=true` turns the whole thing into a no-op, and a
|
|
83
|
+
misconfigured exporter degrades to "telemetry doesn't work" rather than crashing the host process.
|
|
84
|
+
|
|
85
|
+
### Where does it go?
|
|
86
|
+
|
|
87
|
+
| `TELEMETRY_DESTINATION` | Output |
|
|
88
|
+
|---|---|
|
|
89
|
+
| `file` (default) | `$TELEMETRY_FILE_DIR/{traces,metrics,logs}.jsonl` (default `./telemetry`), one OTLP/JSON request per line |
|
|
90
|
+
| `otlp` | OTLP/gRPC to `OTEL_EXPORTER_OTLP_ENDPOINT` (default `http://localhost:4317`) |
|
|
91
|
+
|
|
92
|
+
Full contract: [docs/otel.md → Local file mode](../../docs/otel.md#local-file-mode-the-default).
|
|
93
|
+
|
|
94
|
+
## Reading telemetry back (`query` extra)
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
from ko_observability.query import get_query_client
|
|
98
|
+
|
|
99
|
+
client = get_query_client()
|
|
100
|
+
traces = client.list_traces(limit=20)
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Scoped to the caller's own `service_name` by default. See [`docs/query.md`](../../docs/query.md).
|
|
104
|
+
|
|
105
|
+
## Docs
|
|
106
|
+
|
|
107
|
+
Full documentation — env vars, the logs/metrics pipelines, semantic conventions, and the
|
|
108
|
+
Collector deployment this SDK ships telemetry to — lives in the repository:
|
|
109
|
+
|
|
110
|
+
https://github.com/Komus-ai/nexora-observability/tree/main/docs
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
ko_observability/__init__.py,sha256=NEE95_51mzhAltbETsHCPfmcKIGs72ZBxJgduxDpeWM,359
|
|
2
|
+
ko_observability/client.py,sha256=xGu2TBeXyMgeQ2XGsw6nyfGAG_UBkowUduXIAP2gRQI,2224
|
|
3
|
+
ko_observability/exporters.py,sha256=EW2lEaDwc5foMzxWjOLSQzEXQavldKfQiGNrtvj6I5M,7489
|
|
4
|
+
ko_observability/otel.py,sha256=6hcA0-NnD-dQNFeiHHB0RoULBvCdCQ2jYETCToEVaHs,9409
|
|
5
|
+
ko_observability/otlp_config.py,sha256=Y2XNdHNOdiFklJajjhHuTOkRjzXgpX60hXvp24BfwJM,3093
|
|
6
|
+
ko_observability/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
+
ko_observability/query.py,sha256=jK7C65WP7HRKSwqgjdbdBLP5wGBH2JUwTpMCQQLVb6k,16388
|
|
8
|
+
ko_observability/structlog_bridge.py,sha256=5OMF2WAxC1KPd4ugYHlpGWq1oQ3R9CtOLtKwjlpFdz4,4471
|
|
9
|
+
ko_observability-0.2.0.dist-info/METADATA,sha256=yhfqlHSi1AgphkKccInrn3csoSW9gpkaW0r3ZWQPQm4,4088
|
|
10
|
+
ko_observability-0.2.0.dist-info/WHEEL,sha256=THafob7ofN-NsuMN7Mg4qZyHaQI7KkD-QlcQatYhXPo,87
|
|
11
|
+
ko_observability-0.2.0.dist-info/RECORD,,
|