agent-observatory-cli 0.1.0__py3-none-any.whl

observatory/metrics.py

@@ -0,0 +1,281 @@
+"""Normalize OTLP metric data points from JSON and HTTP/protobuf."""
+from __future__ import annotations
+
+import base64
+import math
+import re
+import struct
+
+from .core import _attrs
+from .otlp_proto import _fields, _nested, _attributes, _text, _kv, _signed
+
+KINDS = {"gauge": 5, "sum": 7, "histogram": 9,
+         "exponentialHistogram": 10, "summary": 11}
+
+
+def _json_safe(value):
+    if isinstance(value, float) and not math.isfinite(value):
+        return str(value)
+    if isinstance(value, list):
+        return [_json_safe(item) for item in value]
+    if isinstance(value, dict):
+        return {key: _json_safe(item) for key, item in value.items()}
+    return value
+
+
+def _normalize(service, resource, scope, metric, kind, temporality, point):
+    timestamp = int(point.get("timeUnixNano") or 0)
+    if timestamp <= 0:
+        raise ValueError("metric point requires timeUnixNano")
+    start = int(point.get("startTimeUnixNano") or 0)
+    value = point.get("asDouble", point.get("asInt"))
+    if value is not None:
+        value = float(value) if "asDouble" in point else int(value)
+    data = {key: item for key, item in point.items() if key not in
+            ("timeUnixNano", "startTimeUnixNano", "attributes", "asDouble", "asInt")}
+    if "exemplars" in point:
+        data["exemplars"] = _normalize_exemplars(point.get("exemplars"))
+    if value is not None:
+        data["value"] = value
+    return {"service": service, "resource_attributes": _json_safe(resource), "scope_name": scope,
+            "name": str(metric.get("name") or "unnamed"),
+            "description": str(metric.get("description") or ""),
+            "unit": str(metric.get("unit") or ""), "kind": kind,
+            "temporality": temporality, "time_unix_nano": timestamp,
+            "start_unix_nano": start, "attributes": _json_safe(_attrs(point.get("attributes"))),
+            "data": _json_safe(data)}
+
+
+
+def _normalize_exemplars(items):
+    result = []
+    for item in items or []:
+        if not isinstance(item, dict):
+            continue
+        trace_id = item.get("trace_id", item.get("traceId"))
+        span_id = item.get("span_id", item.get("spanId"))
+        if isinstance(trace_id, bytes):
+            trace_id = trace_id.hex()
+        if isinstance(span_id, bytes):
+            span_id = span_id.hex()
+        trace_id = str(trace_id).lower() if trace_id is not None else ""
+        span_id = str(span_id).lower() if span_id is not None else ""
+        if trace_id and not re.fullmatch(r"[0-9a-f]{32}", trace_id):
+            trace_id = ""
+        if span_id and not re.fullmatch(r"[0-9a-f]{16}", span_id):
+            span_id = ""
+        try:
+            time_ns = max(0, int(item.get("time_unix_nano", item.get("timeUnixNano", 0))))
+        except (TypeError, ValueError, OverflowError):
+            time_ns = 0
+        value = item.get("value", item.get("asDouble", item.get("asInt")))
+        attrs = item.get("filtered_attributes", item.get("filteredAttributes", {}))
+        if isinstance(attrs, list):
+            attrs = _attrs(attrs)
+        result.append({"trace_id": trace_id or None, "span_id": span_id or None,
+                       "time_unix_nano": time_ns, "value": _json_safe(value),
+                       "filtered_attributes": _json_safe(attrs if isinstance(attrs, dict) else {})})
+    return result
+
+
+def _exemplar(raw, depth):
+    exemplar = {}
+    filtered = {}
+    for number, wire, value in _nested(raw, depth):
+        if number == 2 and wire == 1:
+            exemplar["time_unix_nano"] = value
+        elif number == 3 and wire == 1:
+            exemplar["value"] = _double(value)
+        elif number == 4 and wire == 2:
+            exemplar["span_id"] = value.hex()
+        elif number == 5 and wire == 2:
+            exemplar["trace_id"] = value.hex()
+        elif number == 6 and wire == 1:
+            exemplar["value"] = _signed(value)
+        elif number == 7 and wire == 2:
+            key, item = _kv(value, depth + 1)
+            if key:
+                filtered[key] = item
+    if filtered:
+        exemplar["filtered_attributes"] = filtered
+    return exemplar
+
+def normalize_otlp_metrics(body):
+    result = []
+    for group in body.get("resourceMetrics", []):
+        resource = _attrs(group.get("resource", {}).get("attributes"))
+        service = str(resource.get("service.name") or "unknown")
+        for scope in group.get("scopeMetrics", []):
+            scope_name = str(scope.get("scope", {}).get("name") or "")
+            for metric in scope.get("metrics", []):
+                matched = False
+                for kind in KINDS:
+                    if kind in metric:
+                        matched = True
+                        payload = metric[kind]
+                        temporality = int(payload.get("aggregationTemporality") or 0)
+                        result.extend(_normalize(service, resource, scope_name, metric,
+                                                  kind, temporality, point)
+                                      for point in payload.get("dataPoints", []))
+                        break
+                if not matched:
+                    raise ValueError(f"metric {metric.get('name', 'unnamed')} has no supported data type")
+    return result
+
+
+def _double(value):
+    return struct.unpack("<d", value.to_bytes(8, "little"))[0]
+
+
+def _packed_fixed64(value):
+    if len(value) % 8:
+        raise ValueError("invalid packed fixed64 array")
+    return [int.from_bytes(value[i:i + 8], "little") for i in range(0, len(value), 8)]
+
+
+def _point(raw, kind, depth):
+    attrs_field = 1 if kind == "exponentialHistogram" else 9 if kind == "histogram" else 7
+    point = {"attributes": []}
+    buckets = {"positive": None, "negative": None}
+    exemplar_field = 5 if kind in ("gauge", "sum") else 8 if kind == "histogram" else 11 if kind == "exponentialHistogram" else None
+    for number, wire, value in _nested(raw, depth):
+        if number == exemplar_field and wire == 2:
+            point.setdefault("exemplars", []).append(_exemplar(value, depth + 1))
+        elif number == attrs_field and wire == 2:
+            key, item = _kv(value, depth + 1)
+            if key:
+                point["attributes"].append({"key": key, "value": _json_any(item)})
+        elif number == 2 and wire == 1:
+            point["startTimeUnixNano"] = value
+        elif number == 3 and wire == 1:
+            point["timeUnixNano"] = value
+        elif kind in ("gauge", "sum"):
+            if number == 4 and wire == 1:
+                point["asDouble"] = _double(value)
+            elif number == 6 and wire == 1:
+                point["asInt"] = _signed(value)
+        elif kind in ("histogram", "exponentialHistogram", "summary"):
+            if number == 4 and wire == 1:
+                point["count"] = value
+            elif number == 5 and wire == 1:
+                point["sum"] = _double(value)
+            elif kind == "histogram":
+                if number == 6 and wire == 1:
+                    point.setdefault("bucketCounts", []).append(value)
+                elif number == 6 and wire == 2:
+                    point.setdefault("bucketCounts", []).extend(_packed_fixed64(value))
+                elif number == 7 and wire == 1:
+                    point.setdefault("explicitBounds", []).append(_double(value))
+                elif number == 7 and wire == 2:
+                    point.setdefault("explicitBounds", []).extend(_double(v) for v in _packed_fixed64(value))
+                elif number in (11, 12) and wire == 1:
+                    point["min" if number == 11 else "max"] = _double(value)
+            elif kind == "exponentialHistogram":
+                if number == 6 and wire == 0:
+                    point["scale"] = (value >> 1) ^ -(value & 1)
+                elif number == 7 and wire == 1:
+                    point["zeroCount"] = value
+                elif number in (8, 9) and wire == 2:
+                    buckets["positive" if number == 8 else "negative"] = _exp_buckets(value, depth + 1)
+                elif number in (12, 13, 14) and wire == 1:
+                    point[{12: "min", 13: "max", 14: "zeroThreshold"}[number]] = _double(value)
+            elif kind == "summary" and number == 6 and wire == 2:
+                quantile = {}
+                for field, typ, val in _nested(value, depth + 1):
+                    if field in (1, 2) and typ == 1:
+                        quantile["quantile" if field == 1 else "value"] = _double(val)
+                point.setdefault("quantileValues", []).append(quantile)
+    if kind == "exponentialHistogram":
+        point.update({key: value for key, value in buckets.items() if value is not None})
+    return point
+
+
+def _json_any(value):
+    if isinstance(value, str):
+        return {"stringValue": value}
+    if isinstance(value, bool):
+        return {"boolValue": value}
+    if isinstance(value, int):
+        return {"intValue": value}
+    if isinstance(value, float):
+        return {"doubleValue": value}
+    if isinstance(value, bytes):
+        return {"bytesValue": base64.b64encode(value).decode()}
+    if isinstance(value, list):
+        return {"arrayValue": {"values": [_json_any(v) for v in value]}}
+    if isinstance(value, dict):
+        return {"kvlistValue": {"values": [{"key": k, "value": _json_any(v)} for k, v in value.items()]}}
+    return {}
+
+
+def _exp_buckets(raw, depth):
+    result = {"offset": 0, "bucketCounts": []}
+    for number, wire, value in _nested(raw, depth):
+        if number == 1 and wire == 0:
+            result["offset"] = (value >> 1) ^ -(value & 1)
+        elif number == 2 and wire == 0:
+            result["bucketCounts"].append(value)
+        elif number == 2 and wire == 2:
+            result["bucketCounts"].extend(_unpack_varints(value))
+    return result
+
+
+def _unpack_varints(raw):
+    from .otlp_proto import _varint
+    data = memoryview(raw)
+    pos = 0
+    values = []
+    while pos < len(data):
+        value, pos = _varint(data, pos)
+        values.append(value)
+    return values
+
+
+def _metric(raw, service, resource, scope, depth):
+    metric = {"name": "", "description": "", "unit": ""}
+    payload = None
+    kind = None
+    for number, wire, value in _nested(raw, depth):
+        if number in (1, 2, 3) and wire == 2:
+            metric[{1: "name", 2: "description", 3: "unit"}[number]] = _text(value)
+        elif number in KINDS.values() and wire == 2:
+            kind = next(name for name, field in KINDS.items() if field == number)
+            payload = value
+    if payload is None:
+        raise ValueError(f"metric {metric['name'] or 'unnamed'} has no supported data type")
+    points = []
+    temporality = 0
+    for number, wire, value in _nested(payload, depth + 1):
+        if number == 1 and wire == 2:
+            points.append(_point(value, kind, depth + 2))
+        elif number == 2 and wire == 0:
+            temporality = value
+    return [_normalize(service, resource, scope, metric, kind, temporality, point) for point in points]
+
+
+def decode_metric_request(raw):
+    result = []
+    for number, wire, group in _fields(raw):
+        if number != 1 or wire != 2:
+            continue
+        resource = {}
+        scopes = []
+        for field, kind, value in _nested(group, 1):
+            if field == 1 and kind == 2:
+                resource = _attributes(value, 2)
+            elif field == 2 and kind == 2:
+                scopes.append(value)
+        service = str(resource.get("service.name") or "unknown")
+        for scope in scopes:
+            scope_name = ""
+            metrics = []
+            for field, kind, value in _nested(scope, 2):
+                if field == 1 and kind == 2:
+                    for key, typ, item in _nested(value, 3):
+                        if key == 1 and typ == 2:
+                            scope_name = _text(item)
+                elif field == 2 and kind == 2:
+                    metrics.append(value)
+            for metric in metrics:
+                result.extend(_metric(metric, service, resource, scope_name, 3))
+    return result

observatory/otlp_proto.py

@@ -0,0 +1,210 @@
+"""Decode OTLP/HTTP protobuf trace requests without a protobuf runtime.
+
+Field numbers follow opentelemetry-proto's collector/trace, trace, resource,
+and common v1 messages. Unknown fields are skipped as protobuf requires.
+This module handles the trace signal only; it is not a general protobuf library.
+"""
+from __future__ import annotations
+
+import base64
+import struct
+
+from .core import normalize_native
+
+MAX_DEPTH = 16
+
+
+def _varint(data: memoryview, pos: int):
+    value = 0
+    for shift in range(0, 70, 7):
+        if pos >= len(data):
+            raise ValueError("truncated protobuf varint")
+        byte = data[pos]
+        pos += 1
+        value |= (byte & 0x7f) << shift
+        if byte < 128:
+            if value >= 1 << 64:
+                raise ValueError("protobuf varint exceeds uint64")
+            return value, pos
+    raise ValueError("protobuf varint exceeds 10 bytes")
+
+
+def _fields(raw: bytes, depth=0):
+    if depth > MAX_DEPTH:
+        raise ValueError("protobuf nesting is too deep")
+    data = memoryview(raw)
+    pos = 0
+    while pos < len(data):
+        tag, pos = _varint(data, pos)
+        number, wire = tag >> 3, tag & 7
+        if not number:
+            raise ValueError("invalid protobuf field number")
+        if wire == 0:
+            value, pos = _varint(data, pos)
+        elif wire == 1:
+            if len(data) - pos < 8:
+                raise ValueError("truncated protobuf fixed64")
+            value = int.from_bytes(data[pos:pos + 8], "little")
+            pos += 8
+        elif wire == 2:
+            size, pos = _varint(data, pos)
+            if size > len(data) - pos:
+                raise ValueError("truncated protobuf field")
+            value = bytes(data[pos:pos + size])
+            pos += size
+        elif wire == 5:
+            if len(data) - pos < 4:
+                raise ValueError("truncated protobuf fixed32")
+            value = int.from_bytes(data[pos:pos + 4], "little")
+            pos += 4
+        else:
+            raise ValueError(f"unsupported protobuf wire type {wire}")
+        yield number, wire, value
+
+
+def _text(value):
+    return value.decode("utf-8")
+
+
+def _nested(value, depth):
+    return _fields(value, depth + 1)
+
+
+def _signed(value):
+    return value - (1 << 64) if value >= (1 << 63) else value
+
+
+def _any_value(raw, depth):
+    result = None
+    for number, wire, value in _nested(raw, depth):
+        if number == 1 and wire == 2:
+            result = _text(value)
+        elif number == 2 and wire == 0:
+            result = bool(value)
+        elif number == 3 and wire == 0:
+            result = _signed(value)
+        elif number == 4 and wire == 1:
+            result = struct.unpack("<d", value.to_bytes(8, "little"))[0]
+        elif number == 5 and wire == 2:
+            result = [_any_value(v, depth + 2) for n, w, v in _nested(value, depth + 1) if n == 1 and w == 2]
+        elif number == 6 and wire == 2:
+            result = _attributes(value, depth + 1)
+        elif number == 7 and wire == 2:
+            result = base64.b64encode(value).decode("ascii")
+    return result
+
+
+def _kv(raw, depth):
+    key, value = "", None
+    for number, wire, item in _nested(raw, depth):
+        if number == 1 and wire == 2:
+            key = _text(item)
+        elif number == 2 and wire == 2:
+            value = _any_value(item, depth + 1)
+    return key, value
+
+
+def _attributes(raw, depth, field=1):
+    result = {}
+    for number, wire, value in _nested(raw, depth):
+        if number == field and wire == 2:
+            key, item = _kv(value, depth + 1)
+            if key:
+                result[key] = item
+    return result
+
+
+def _event(raw, depth):
+    event = {"name": "event", "time_unix_nano": 0, "attributes": {}}
+    for number, wire, value in _nested(raw, depth):
+        if number == 1 and wire == 1:
+            event["time_unix_nano"] = value
+        elif number == 2 and wire == 2:
+            event["name"] = _text(value)
+        elif number == 3 and wire == 2:
+            key, item = _kv(value, depth + 1)
+            if key:
+                event["attributes"][key] = item
+    return event
+
+
+def _status(raw, depth):
+    code = 0
+    for number, wire, value in _nested(raw, depth):
+        if number == 3 and wire == 0:
+            code = value
+    return {0: "UNSET", 1: "OK", 2: "ERROR"}.get(code, "UNSET")
+
+
+def _link(raw, depth):
+    result = {"trace_id": None, "span_id": None, "attributes": {}}
+    for number, wire, value in _nested(raw, depth):
+        if number == 1 and wire == 2:
+            result["trace_id"] = value.hex()
+        elif number == 2 and wire == 2:
+            result["span_id"] = value.hex()
+        elif number == 4 and wire == 2:
+            key, item = _kv(value, depth + 1)
+            if key:
+                result["attributes"][key] = item
+    return result
+
+
+def _span(raw, service, resource, scope_name, depth):
+    native = {"trace_id": None, "span_id": None, "parent_span_id": None,
+              "name": "unnamed", "start_unix_nano": 0, "end_unix_nano": 0,
+              "status": "UNSET", "attributes": {}, "events": [], "links": [],
+              "resource_attributes": resource, "scope_name": scope_name}
+    for number, wire, value in _nested(raw, depth):
+        if number == 1 and wire == 2:
+            native["trace_id"] = value.hex()
+        elif number == 2 and wire == 2:
+            native["span_id"] = value.hex()
+        elif number == 4 and wire == 2:
+            native["parent_span_id"] = value.hex() if value else None
+        elif number == 5 and wire == 2:
+            native["name"] = _text(value)
+        elif number == 7 and wire == 1:
+            native["start_unix_nano"] = value
+        elif number == 8 and wire == 1:
+            native["end_unix_nano"] = value
+        elif number == 9 and wire == 2:
+            key, item = _kv(value, depth + 1)
+            if key:
+                native["attributes"][key] = item
+        elif number == 11 and wire == 2:
+            native["events"].append(_event(value, depth + 1))
+        elif number == 13 and wire == 2:
+            native["links"].append(_link(value, depth + 1))
+        elif number == 15 and wire == 2:
+            native["status"] = _status(value, depth + 1)
+    return normalize_native(native, service)
+
+
+def decode_trace_request(raw: bytes):
+    """Return normalized spans from an ExportTraceServiceRequest protobuf body."""
+    spans = []
+    for number, wire, resource_spans in _fields(raw):
+        if number != 1 or wire != 2:
+            continue
+        resource = {}
+        scopes = []
+        for field, kind, value in _nested(resource_spans, 1):
+            if field == 1 and kind == 2:
+                resource = _attributes(value, 2)
+            elif field == 2 and kind == 2:
+                scopes.append(value)
+        for scope in scopes:
+            scope_name = ""
+            raw_spans = []
+            for field, kind, value in _nested(scope, 2):
+                if field == 1 and kind == 2:
+                    for scope_field, scope_kind, scope_value in _nested(value, 3):
+                        if scope_field == 1 and scope_kind == 2:
+                            scope_name = _text(scope_value)
+                elif field == 2 and kind == 2:
+                    raw_spans.append(value)
+            for span in raw_spans:
+                spans.append(_span(span, str(resource.get("service.name") or "unknown"),
+                                   resource, scope_name, 3))
+    return spans