spanforge 1.0.0__py3-none-any.whl

spanforge/export/otlp_bridge.py

@@ -0,0 +1,233 @@
+"""spanforge.export.otlp_bridge — Lightweight Span → OTLP dict translation.
+
+Translates :class:`~spanforge._span.Span` instances into the OTLP ``ReadableSpan``
+dict format **without** requiring the ``opentelemetry-sdk`` package.  The produced
+dicts are wire-compatible with the OTLP/JSON specification and can be embedded
+directly into ``resourceSpans[].scopeSpans[].spans`` payloads.
+
+This complements :mod:`spanforge.export.otel_bridge` (which bridges to the OTel
+SDK's ``TracerProvider``) by providing a zero-dependency serialisation path for
+testing, custom exporters, and SDK-less environments.
+
+Usage::
+
+    from spanforge.export.otlp_bridge import span_to_otlp_dict, SpanOTLPBridge
+
+    # Single span → OTLP dict (no OTel SDK required)
+    otlp = span_to_otlp_dict(my_span)
+    print(otlp["name"], otlp["traceId"], otlp["spanId"])
+
+    # Full OTLP resource-spans envelope
+    bridge = SpanOTLPBridge(service_name="my-agent")
+    payload = bridge.to_resource_spans([span1, span2])
+    # → {"resourceSpans": [...]}
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from spanforge._span import Span
+
+__all__ = ["SpanOTLPBridge", "span_to_otlp_dict"]
+
+# OTLP SpanKind integer constants (opentelemetry.proto.trace.v1.Span.SpanKind).
+_OTLP_SPAN_KIND_UNSPECIFIED = 0
+_OTLP_SPAN_KIND_INTERNAL = 1
+_OTLP_SPAN_KIND_SERVER = 2
+_OTLP_SPAN_KIND_CLIENT = 3
+_OTLP_SPAN_KIND_PRODUCER = 4
+_OTLP_SPAN_KIND_CONSUMER = 5
+
+# OTLP Status codes (opentelemetry.proto.trace.v1.Status.StatusCode).
+_OTLP_STATUS_UNSET = 0
+_OTLP_STATUS_OK = 1
+_OTLP_STATUS_ERROR = 2
+
+_SCOPE_NAME = "spanforge"
+_SCOPE_VERSION = "2.0.0"
+
+
+# ---------------------------------------------------------------------------
+# Attribute helpers
+# ---------------------------------------------------------------------------
+
+
+def _string_attr(key: str, value: str) -> dict[str, Any]:
+    return {"key": key, "value": {"stringValue": value}}
+
+
+def _int_attr(key: str, value: int) -> dict[str, Any]:
+    return {"key": key, "value": {"intValue": str(value)}}
+
+
+def _bool_attr(key: str, value: bool) -> dict[str, Any]:
+    return {"key": key, "value": {"boolValue": value}}
+
+
+def _double_attr(key: str, value: float) -> dict[str, Any]:
+    return {"key": key, "value": {"doubleValue": value}}
+
+
+def _to_otlp_attr(key: str, value: Any) -> dict[str, Any]:
+    """Convert a single key-value pair to an OTLP-format attribute dict."""
+    if isinstance(value, bool):
+        return _bool_attr(key, value)
+    if isinstance(value, int):
+        return _int_attr(key, value)
+    if isinstance(value, float):
+        return _double_attr(key, value)
+    return _string_attr(key, str(value))
+
+
+# ---------------------------------------------------------------------------
+# span_to_otlp_dict
+# ---------------------------------------------------------------------------
+
+
+def span_to_otlp_dict(span: Span) -> dict[str, Any]:
+    """Translate a :class:`~spanforge._span.Span` to an OTLP span dict.
+
+    The returned dict conforms to the OTLP/JSON ``Span`` protobuf shape
+    (``opentelemetry.proto.trace.v1.Span``).  All nanosecond timestamps are
+    serialised as **strings** per the OTLP/JSON spec.
+
+    Args:
+        span: A :class:`~spanforge._span.Span` instance (open or closed).
+              When ``end_ns`` is ``None`` the current time is substituted.
+
+    Returns:
+        OTLP-format ``dict`` ready to embed in a ``resourceSpans`` payload.
+    """
+    import time
+
+    start_ns = span.start_ns
+    end_ns = span.end_ns if span.end_ns is not None else time.time_ns()
+
+    # Build attributes list following gen_ai.* semantic conventions.
+    attrs: list[dict[str, Any]] = []
+    if span.model:
+        attrs.append(_string_attr("gen_ai.request.model", span.model))
+    if span.operation:
+        attrs.append(_string_attr("gen_ai.operation.name", str(span.operation)))
+    if span.temperature is not None:
+        attrs.append(_double_attr("gen_ai.request.temperature", span.temperature))
+    if span.top_p is not None:
+        attrs.append(_double_attr("gen_ai.request.top_p", span.top_p))
+    if span.max_tokens is not None:
+        attrs.append(_int_attr("gen_ai.request.max_tokens", span.max_tokens))
+    if span.token_usage is not None:
+        tu = span.token_usage
+        if tu.input_tokens is not None:
+            attrs.append(_int_attr("gen_ai.usage.input_tokens", tu.input_tokens))
+        if tu.output_tokens is not None:
+            attrs.append(_int_attr("gen_ai.usage.output_tokens", tu.output_tokens))
+        if tu.total_tokens is not None:
+            attrs.append(_int_attr("gen_ai.usage.total_tokens", tu.total_tokens))
+    if span.error:
+        attrs.append(_string_attr("exception.message", span.error))
+    if span.error_type:
+        attrs.append(_string_attr("exception.type", span.error_type))
+    if span.error_category:
+        attrs.append(_string_attr("spanforge.error.category", span.error_category))
+    for k, v in (span.attributes or {}).items():
+        attrs.append(_to_otlp_attr(str(k), v))
+
+    # Status code mapping.
+    status_code = _OTLP_STATUS_ERROR if span.status == "error" else _OTLP_STATUS_OK
+    otlp_status: dict[str, Any] = {"code": status_code}
+    if span.error:
+        otlp_status["message"] = span.error
+
+    result: dict[str, Any] = {
+        "traceId": span.trace_id,
+        "spanId": span.span_id,
+        "name": span.name,
+        "kind": _OTLP_SPAN_KIND_CLIENT,
+        "startTimeUnixNano": str(start_ns),
+        "endTimeUnixNano": str(end_ns),
+        "attributes": attrs,
+        "status": otlp_status,
+        "events": [],
+        "links": [],
+    }
+
+    if span.parent_span_id:
+        result["parentSpanId"] = span.parent_span_id
+
+    # Translate SpanEvent list to OTLP events.
+    for ev in span.events or []:
+        ev_attrs = [_to_otlp_attr(str(k), v) for k, v in (ev.metadata or {}).items()]
+        result["events"].append(
+            {
+                "name": ev.name,
+                "timeUnixNano": str(start_ns),  # SpanEvent has no own timestamp; use span start
+                "attributes": ev_attrs,
+            }
+        )
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# SpanOTLPBridge
+# ---------------------------------------------------------------------------
+
+
+class SpanOTLPBridge:
+    """Assembles OTLP ``resourceSpans`` payloads from :class:`~spanforge._span.Span` lists.
+
+    Usage::
+
+        bridge = SpanOTLPBridge(service_name="my-agent", service_version="1.0.0")
+        payload = bridge.to_resource_spans([span1, span2])
+        # → {"resourceSpans": [...]}
+
+    Args:
+        service_name:    Value for the ``service.name`` resource attribute.
+        service_version: Optional value for the ``service.version`` resource attribute.
+    """
+
+    def __init__(
+        self,
+        service_name: str = "spanforge",
+        service_version: str | None = None,
+    ) -> None:
+        self.service_name = service_name
+        self.service_version = service_version
+
+    def to_resource_spans(self, spans: list[Span]) -> dict[str, Any]:
+        """Build a complete OTLP/JSON ``resourceSpans`` envelope from *spans*.
+
+        Args:
+            spans: List of :class:`~spanforge._span.Span` objects to serialise.
+
+        Returns:
+            ``{"resourceSpans": [...]}`` dict for JSON serialisation or forwarding
+            to an OTLP collector.
+        """
+        resource_attrs: list[dict[str, Any]] = [
+            _string_attr("service.name", self.service_name),
+        ]
+        if self.service_version:
+            resource_attrs.append(_string_attr("service.version", self.service_version))
+
+        otlp_spans = [span_to_otlp_dict(s) for s in spans]
+
+        return {
+            "resourceSpans": [
+                {
+                    "resource": {"attributes": resource_attrs},
+                    "scopeSpans": [
+                        {
+                            "scope": {
+                                "name": _SCOPE_NAME,
+                                "version": _SCOPE_VERSION,
+                            },
+                            "spans": otlp_spans,
+                        }
+                    ],
+                }
+            ]
+        }

spanforge/export/redis_backend.py

@@ -0,0 +1,282 @@
+"""spanforge.export.redis_backend — Redis-backed event cache and exporter.
+
+Stores serialised spanforge events in a Redis stream (``XADD``) or a capped
+list, making exported events available to multiple consumers and providing a
+persistent replay buffer.
+
+**Optional dependency**: requires ``redis >= 4.0``.  Install via::
+
+    pip install "spanforge[redis]"
+
+Usage::
+
+    import asyncio
+    import spanforge
+    from spanforge.export.redis_backend import RedisExporter
+
+    async def main():
+        exporter = RedisExporter(url="redis://localhost:6379", stream_key="spanforge:events")
+        spanforge.configure(exporter="redis", endpoint="redis://localhost:6379")
+
+        async with exporter:
+            with spanforge.span("my-llm-call") as span:
+                span.set_model(model="gpt-4o", system="openai")
+                span.set_status("ok")
+
+    asyncio.run(main())
+
+Reading events back::
+
+    from spanforge.export.redis_backend import RedisEventReader
+    reader = RedisEventReader(url="redis://localhost:6379", stream_key="spanforge:events")
+    async for event_dict in reader.read(count=100):
+        print(event_dict)
+
+Environment variables::
+
+    SPANFORGE_REDIS_URL          Redis connection URL (default: redis://localhost:6379)
+    SPANFORGE_REDIS_STREAM_KEY   Stream key name (default: spanforge:events)
+    SPANFORGE_REDIS_MAX_LEN      Max stream length / MAXLEN trim (default: 100000)
+    SPANFORGE_REDIS_TTL_SECONDS  Per-entry TTL; 0 = no TTL (default: 0)
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
+    from spanforge.event import Event
+
+__all__ = ["RedisEventReader", "RedisExporter"]
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_URL = "redis://localhost:6379"
+_DEFAULT_STREAM_KEY = "spanforge:events"
+_DEFAULT_MAX_LEN = 100_000
+
+
+def _require_redis() -> Any:
+    """Import and return the redis.asyncio module, raising ImportError with hint if absent."""
+    try:
+        import redis.asyncio as aioredis
+    except ImportError as exc:
+        raise ImportError(
+            "The Redis exporter requires the 'redis' package. "
+            'Install it with: pip install "spanforge[redis]"'
+        ) from exc
+    else:
+        return aioredis
+
+
+class RedisExporter:
+    """Async exporter that writes spanforge events into a Redis Stream.
+
+    Uses Redis Streams (``XADD``) with approximate ``MAXLEN`` trimming to
+    provide a durable, multi-consumer event buffer.
+
+    Args:
+        url:           Redis connection URL (``redis://``, ``rediss://``, or
+                       ``unix://`` socket path).
+        stream_key:    Redis stream key name.
+        max_len:       Approximate maximum number of entries retained in the stream.
+                       Older entries are trimmed automatically.
+        ttl_seconds:   When > 0, the Redis key TTL is refreshed on every write
+                       (sliding expiry).  ``0`` disables TTL management.
+        decode_responses: Whether to decode Redis responses to Python strings
+                       (default: False — bytes are returned for payloads).
+
+    Thread / coroutine safety:
+        The underlying ``redis.asyncio`` client is coroutine-safe.  Concurrent
+        calls to :meth:`export` are safe without external locking.
+    """
+
+    def __init__(
+        self,
+        url: str = _DEFAULT_URL,
+        stream_key: str = _DEFAULT_STREAM_KEY,
+        max_len: int = _DEFAULT_MAX_LEN,
+        ttl_seconds: int = 0,
+        *,
+        decode_responses: bool = False,
+    ) -> None:
+        self._url = url or os.environ.get("SPANFORGE_REDIS_URL", _DEFAULT_URL)
+        self._stream_key = stream_key or os.environ.get(
+            "SPANFORGE_REDIS_STREAM_KEY", _DEFAULT_STREAM_KEY
+        )
+        max_len_env = int(os.environ.get("SPANFORGE_REDIS_MAX_LEN", str(_DEFAULT_MAX_LEN)))
+        self._max_len = max_len if max_len != _DEFAULT_MAX_LEN else max_len_env
+        ttl_env = int(os.environ.get("SPANFORGE_REDIS_TTL_SECONDS", "0"))
+        self._ttl = ttl_seconds if ttl_seconds != 0 else ttl_env
+        self._decode = decode_responses
+        self._client: object | None = None
+
+    # ------------------------------------------------------------------
+    # Async context manager
+    # ------------------------------------------------------------------
+
+    async def __aenter__(self) -> RedisExporter:
+        await self._connect()
+        return self
+
+    async def __aexit__(self, *_: object) -> None:
+        await self.close()
+
+    # ------------------------------------------------------------------
+    # Internal
+    # ------------------------------------------------------------------
+
+    async def _connect(self) -> None:
+        if self._client is not None:
+            return
+        aioredis = _require_redis()
+        self._client = aioredis.from_url(self._url, decode_responses=self._decode)
+
+    # ------------------------------------------------------------------
+    # Public interface
+    # ------------------------------------------------------------------
+
+    async def export(self, event: Event) -> None:
+        """Serialize *event* and write it to the Redis stream.
+
+        Args:
+            event: spanforge :class:`~spanforge.event.Event` instance.
+
+        Raises:
+            ImportError: If the ``redis`` package is not installed.
+            redis.RedisError: On connection or write failure.
+        """
+        if self._client is None:
+            await self._connect()
+
+        payload = json.dumps(event.to_dict(), separators=(",", ":"), default=str)
+        fields = {
+            b"data" if not self._decode else "data": (
+                payload.encode() if not self._decode else payload
+            ),
+            b"event_id" if not self._decode else "event_id": (
+                event.event_id.encode() if not self._decode else event.event_id
+            ),
+            b"event_type" if not self._decode else "event_type": (
+                event.event_type.encode() if not self._decode else event.event_type
+            ),
+        }
+        assert self._client is not None  # nosec B101
+        await self._client.xadd(  # type: ignore[attr-defined]
+            self._stream_key,
+            fields,
+            maxlen=self._max_len,
+            approximate=True,
+        )
+        if self._ttl > 0:
+            await self._client.expire(self._stream_key, self._ttl)  # type: ignore[attr-defined]
+
+    async def export_batch(self, events: list[Event]) -> None:
+        """Write multiple events in a single pipeline round-trip."""
+        if not events:
+            return
+        if self._client is None:
+            await self._connect()
+
+        assert self._client is not None  # nosec B101
+        pipe = self._client.pipeline(transaction=False)  # type: ignore[attr-defined]
+        for event in events:
+            payload = json.dumps(event.to_dict(), separators=(",", ":"), default=str)
+            fields = {
+                b"data" if not self._decode else "data": (
+                    payload.encode() if not self._decode else payload
+                ),
+                b"event_id" if not self._decode else "event_id": (
+                    event.event_id.encode() if not self._decode else event.event_id
+                ),
+            }
+            pipe.xadd(self._stream_key, fields, maxlen=self._max_len, approximate=True)
+        if self._ttl > 0:
+            pipe.expire(self._stream_key, self._ttl)
+        await pipe.execute()
+
+    async def close(self) -> None:
+        """Close the underlying Redis connection."""
+        if self._client is not None:
+            try:
+                await self._client.aclose()  # type: ignore[attr-defined]
+            except Exception as _err:
+                logger.debug("Redis close error: %s", _err)
+            finally:
+                self._client = None
+
+    async def flush(self) -> None:
+        """No-op — Redis writes are immediately durable."""
+
+
+class RedisEventReader:
+    """Read spanforge events back from a Redis Stream.
+
+    Args:
+        url:        Redis connection URL.
+        stream_key: Redis stream key name.
+
+    Example::
+
+        reader = RedisEventReader(url="redis://localhost:6379")
+        async for raw in reader.read(count=50, last_id="0"):
+            print(raw)
+    """
+
+    def __init__(
+        self,
+        url: str = _DEFAULT_URL,
+        stream_key: str = _DEFAULT_STREAM_KEY,
+    ) -> None:
+        self._url = url
+        self._stream_key = stream_key
+        self._client: object | None = None
+
+    async def __aenter__(self) -> RedisEventReader:
+        aioredis = _require_redis()
+        self._client = aioredis.from_url(self._url, decode_responses=True)
+        return self
+
+    async def __aexit__(self, *_: object) -> None:
+        if self._client is not None:
+            await self._client.aclose()  # type: ignore[attr-defined]
+
+    async def read(
+        self,
+        count: int = 100,
+        last_id: str = "0",
+    ) -> AsyncIterator[dict[str, Any]]:
+        """Yield event dicts from the stream starting after *last_id*.
+
+        Args:
+            count:   Maximum number of entries to fetch per call.
+            last_id: Stream entry ID to start reading from (exclusive).
+                     ``"0"`` reads from the beginning.
+                     ``"$"`` reads only new entries.
+
+        Yields:
+            Raw event dicts (deserialised from the ``data`` field).
+        """
+        if self._client is None:
+            raise RuntimeError("Use 'async with RedisEventReader(...) as reader:' first")
+
+        entries = await self._client.xread(  # type: ignore[attr-defined]
+            {self._stream_key: last_id}, count=count
+        )
+        if not entries:
+            return
+        for _stream_name, records in entries:
+            for _entry_id, fields in records:
+                raw = fields.get("data") or fields.get(b"data", b"{}")
+                if isinstance(raw, bytes):
+                    raw = raw.decode()
+                try:
+                    yield json.loads(raw)
+                except json.JSONDecodeError:
+                    logger.warning("RedisEventReader: could not deserialise entry")
+                    continue

spanforge/export/siem_schema.py

@@ -0,0 +1,98 @@
+"""SIEM-friendly schema normalization for SpanForge events.
+
+Provides one canonical event mapping that downstream SIEM exporters can reuse
+so Splunk, syslog/CEF, and JSON-based audit sinks see the same core fields.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from spanforge.event import Event
+
+__all__ = ["event_to_siem_record", "severity_from_event"]
+
+_SEVERITY_MAP: dict[str, int] = {
+    "alert": 1,
+    "error": 3,
+    "warn": 4,
+    "warning": 4,
+    "info": 6,
+    "debug": 7,
+    "trace": 7,
+}
+
+
+def severity_from_event(event: Event) -> int:
+    """Map the event_type prefix to a syslog-style severity."""
+    prefix = event.event_type.split(".")[0].lower()
+    return _SEVERITY_MAP.get(prefix, 6)
+
+
+def event_to_siem_record(event: Event) -> dict[str, Any]:
+    """Return a normalized SIEM record for one SpanForge event."""
+    payload = _safe_payload(getattr(event, "payload", {}))
+    tags = _safe_tags(getattr(event, "tags", None))
+    record = {
+        "event_id": getattr(event, "event_id", None),
+        "event_type": getattr(event, "event_type", None),
+        "schema_version": getattr(event, "schema_version", None),
+        "timestamp": getattr(event, "timestamp", None),
+        "source": getattr(event, "source", None),
+        "trace_id": getattr(event, "trace_id", None),
+        "span_id": getattr(event, "span_id", None),
+        "parent_span_id": getattr(event, "parent_span_id", None),
+        "org_id": getattr(event, "org_id", None),
+        "team_id": getattr(event, "team_id", None),
+        "actor_id": getattr(event, "actor_id", None),
+        "session_id": getattr(event, "session_id", None),
+        "siem": {
+            "schema": "spanforge.event.v1",
+            "category": _category_for_event_type(str(getattr(event, "event_type", ""))),
+            "severity": severity_from_event(event),
+        },
+        "payload": payload,
+        "tags": tags,
+    }
+    normalized = json.loads(json.dumps(record, sort_keys=True, default=str))
+    return {key: value for key, value in normalized.items() if value not in (None, {})}
+
+
+def _category_for_event_type(event_type: str) -> str:
+    if ".policy." in event_type:
+        return "policy"
+    if ".audit." in event_type:
+        return "audit"
+    if ".scope." in event_type:
+        return "scope"
+    if ".rbac." in event_type:
+        return "rbac"
+    if ".rag." in event_type or ".ground" in event_type:
+        return "grounding"
+    if ".lineage." in event_type:
+        return "lineage"
+    if ".tool" in event_type:
+        return "tool"
+    if ".trace." in event_type or event_type.startswith("trace."):
+        return "trace"
+    return "application"
+
+
+def _safe_payload(payload: Any) -> dict[str, Any]:
+    if isinstance(payload, dict):
+        return payload
+    try:
+        return dict(payload)
+    except Exception:
+        return {"value": payload}
+
+
+def _safe_tags(tags: Any) -> dict[str, Any]:
+    if tags is None:
+        return {}
+    try:
+        value = tags.to_dict() if hasattr(tags, "to_dict") else dict(tags)
+        return value if isinstance(value, dict) else {}
+    except Exception:
+        return {}