icdev 0.0.3__py3-none-any.whl

tools/observability/sqlite_tracer.py

@@ -0,0 +1,360 @@
+#!/usr/bin/env python3
+# CUI // SP-CTI
+"""SQLite Trace Backend — Air-gapped span storage (D280).
+
+Writes spans to the `otel_spans` table in icdev.db.
+Designed for IL5/IL6 environments with no network access to collectors.
+
+Features:
+  - Append-only span storage (D6 compliance)
+  - Hierarchical span nesting via parent_span_id
+  - contextvar-based active span tracking
+  - Thread-safe SQLite writes
+  - Span buffering with configurable flush interval
+
+Usage:
+    from tools.observability.sqlite_tracer import SQLiteTracer
+    tracer = SQLiteTracer()
+
+    with tracer.start_span("my_op", attributes={"key": "val"}) as span:
+        span.set_attribute("result", "ok")
+"""
+
+import contextvars
+import json
+import logging
+import sqlite3
+import threading
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from tools.observability.tracer import NullSpan, Span, Tracer
+from tools.db.storage import get_connection
+DB_PATH = None  # Storage layer handles path resolution (D-DB-20)
+
+logger = logging.getLogger("sparkpilot.observability.sqlite_tracer")
+
+BASE_DIR = Path(__file__).resolve().parent.parent.parent
+
+# Active span tracking via contextvars
+_active_span_var: contextvars.ContextVar[Optional["SQLiteSpan"]] = contextvars.ContextVar(
+    "icdev_active_span", default=None
+)
+
+
+class SQLiteSpan(Span):
+    """Span implementation backed by SQLite (D280)."""
+
+    def __init__(
+        self,
+        name: str,
+        trace_id: str,
+        span_id: str,
+        parent_span_id: Optional[str] = None,
+        kind: str = "INTERNAL",
+        attributes: Optional[Dict[str, Any]] = None,
+        tracer: Optional["SQLiteTracer"] = None,
+    ):
+        self._name = name
+        self._trace_id = trace_id
+        self._span_id = span_id
+        self._parent_span_id = parent_span_id
+        self._kind = kind
+        self._attributes: Dict[str, Any] = attributes or {}
+        self._events: List[Dict[str, Any]] = []
+        self._status_code = "UNSET"
+        self._status_message = ""
+        self._start_time = datetime.now(timezone.utc)
+        self._end_time: Optional[datetime] = None
+        self._duration_ms: int = 0
+        self._tracer = tracer
+        self._ended = False
+
+        # Set as active span
+        self._context_token = _active_span_var.set(self)
+
+    @property
+    def span_id(self) -> str:
+        return self._span_id
+
+    @property
+    def trace_id(self) -> str:
+        return self._trace_id
+
+    @property
+    def parent_span_id(self) -> Optional[str]:
+        return self._parent_span_id
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def kind(self) -> str:
+        return self._kind
+
+    @property
+    def start_time(self) -> datetime:
+        return self._start_time
+
+    @property
+    def end_time(self) -> Optional[datetime]:
+        return self._end_time
+
+    @property
+    def duration_ms(self) -> int:
+        return self._duration_ms
+
+    @property
+    def status_code(self) -> str:
+        return self._status_code
+
+    @property
+    def attributes(self) -> Dict[str, Any]:
+        return dict(self._attributes)
+
+    def set_attribute(self, key: str, value: Any) -> None:
+        if not self._ended:
+            self._attributes[key] = value
+
+    def add_event(self, name: str, attributes: Optional[Dict[str, Any]] = None) -> None:
+        if not self._ended:
+            self._events.append({
+                "name": name,
+                "timestamp": datetime.now(timezone.utc).isoformat(),
+                "attributes": attributes or {},
+            })
+
+    def set_status(self, code: str, message: str = "") -> None:
+        if not self._ended:
+            self._status_code = code
+            self._status_message = message
+
+    def end(self) -> None:
+        if self._ended:
+            return
+        self._ended = True
+        self._end_time = datetime.now(timezone.utc)
+        delta = self._end_time - self._start_time
+        self._duration_ms = int(delta.total_seconds() * 1000)
+
+        # Restore previous active span
+        _active_span_var.set(None)
+
+        # Persist to SQLite
+        if self._tracer:
+            self._tracer._record_span(self)
+
+    def _raw_status_code(self) -> str:
+        return self._status_code
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Serialize span to dictionary for storage/export."""
+        return {
+            "id": self._span_id,
+            "trace_id": self._trace_id,
+            "parent_span_id": self._parent_span_id,
+            "name": self._name,
+            "kind": self._kind,
+            "start_time": self._start_time.isoformat(),
+            "end_time": self._end_time.isoformat() if self._end_time else None,
+            "duration_ms": self._duration_ms,
+            "status_code": self._status_code,
+            "status_message": self._status_message,
+            "attributes": self._attributes,
+            "events": self._events,
+        }
+
+
+class SQLiteTracer(Tracer):
+    """SQLite-backed tracer for air-gapped environments (D280).
+
+    Writes completed spans to the `otel_spans` table in icdev.db.
+    Thread-safe via a lock on write operations.
+    """
+
+    def __init__(
+        self,
+        db_path: Optional[Path] = None,
+        agent_id: Optional[str] = None,
+        project_id: Optional[str] = None,
+        classification: str = "CUI",
+    ):
+        self._db_path = db_path or DEFAULT_DB_PATH
+        self._agent_id = agent_id
+        self._project_id = project_id
+        self._classification = classification
+        self._write_lock = threading.Lock()
+        self._buffer: List[SQLiteSpan] = []
+        self._buffer_size = 10  # Flush after N spans
+
+    def start_span(
+        self,
+        name: str,
+        parent: Optional[Span] = None,
+        kind: str = "INTERNAL",
+        attributes: Optional[Dict[str, Any]] = None,
+    ) -> SQLiteSpan:
+        """Create and start a new SQLite-backed span."""
+        # Inherit trace_id from parent or active span or generate new
+        active = _active_span_var.get()
+        if parent:
+            trace_id = parent.trace_id
+            parent_id = parent.span_id
+        elif active:
+            trace_id = active.trace_id
+            parent_id = active.span_id
+        else:
+            trace_id = uuid.uuid4().hex
+            parent_id = None
+
+        span_id = uuid.uuid4().hex[:16]
+
+        # Merge default attributes
+        attrs = {}
+        if self._agent_id:
+            attrs["sparkpilot.agent_id"] = self._agent_id
+        if self._project_id:
+            attrs["sparkpilot.project_id"] = self._project_id
+        if attributes:
+            attrs.update(attributes)
+
+        return SQLiteSpan(
+            name=name,
+            trace_id=trace_id,
+            span_id=span_id,
+            parent_span_id=parent_id,
+            kind=kind,
+            attributes=attrs,
+            tracer=self,
+        )
+
+    def get_active_span(self) -> Optional[Span]:
+        return _active_span_var.get()
+
+    def _record_span(self, span: SQLiteSpan) -> None:
+        """Record a completed span to the buffer, flush if needed."""
+        self._buffer.append(span)
+        if len(self._buffer) >= self._buffer_size:
+            self.flush()
+
+    def flush(self) -> None:
+        """Write buffered spans to SQLite (thread-safe)."""
+        if not self._buffer:
+            return
+
+        with self._write_lock:
+            spans_to_write = list(self._buffer)
+            self._buffer.clear()
+
+        if not self._db_path.exists():
+            logger.warning("Database not found at %s — spans discarded", self._db_path)
+            return
+
+        try:
+            conn = sqlite3.connect(str(self._db_path), timeout=5)
+            for span in spans_to_write:
+                try:
+                    conn.execute(
+                        """INSERT OR IGNORE INTO otel_spans
+                           (id, trace_id, parent_span_id, name, kind,
+                            start_time, end_time, duration_ms,
+                            status_code, status_message,
+                            attributes, events,
+                            agent_id, project_id, classification)
+                           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+                        (
+                            span.span_id,
+                            span.trace_id,
+                            span.parent_span_id,
+                            span.name,
+                            span.kind,
+                            span.start_time.isoformat(),
+                            span.end_time.isoformat() if span.end_time else None,
+                            span.duration_ms,
+                            span.status_code,
+                            span._status_message,
+                            json.dumps(span.attributes),
+                            json.dumps(span._events),
+                            span.attributes.get("sparkpilot.agent_id", self._agent_id),
+                            span.attributes.get("sparkpilot.project_id", self._project_id),
+                            self._classification,
+                        ),
+                    )
+                except sqlite3.Error as e:
+                    logger.error("Failed to write span %s: %s", span.span_id, e)
+            conn.commit()
+            conn.close()
+        except sqlite3.Error as e:
+            logger.error("SQLite error flushing spans: %s", e)
+
+    def query_spans(
+        self,
+        trace_id: Optional[str] = None,
+        project_id: Optional[str] = None,
+        name: Optional[str] = None,
+        limit: int = 100,
+    ) -> List[Dict[str, Any]]:
+        """Query stored spans from SQLite.
+
+        Args:
+            trace_id: Filter by trace ID.
+            project_id: Filter by project ID.
+            name: Filter by span name (exact match).
+            limit: Maximum results.
+
+        Returns:
+            List of span dictionaries.
+        """
+        if not self._db_path.exists():
+            return []
+
+        conditions = []
+        params = []
+
+        if trace_id:
+            conditions.append("trace_id = ?")
+            params.append(trace_id)
+        if project_id:
+            conditions.append("project_id = ?")
+            params.append(project_id)
+        if name:
+            conditions.append("name = ?")
+            params.append(name)
+
+        where = " AND ".join(conditions) if conditions else "1=1"
+        params.append(limit)
+
+        try:
+            conn = sqlite3.connect(str(self._db_path))
+            rows = conn.execute(
+                f"SELECT * FROM otel_spans WHERE {where} ORDER BY start_time DESC LIMIT ?",
+                params,
+            ).fetchall()
+            conn.close()
+
+            return [
+                {
+                    "id": row["id"],
+                    "trace_id": row["trace_id"],
+                    "parent_span_id": row["parent_span_id"],
+                    "name": row["name"],
+                    "kind": row["kind"],
+                    "start_time": row["start_time"],
+                    "end_time": row["end_time"],
+                    "duration_ms": row["duration_ms"],
+                    "status_code": row["status_code"],
+                    "status_message": row["status_message"],
+                    "attributes": json.loads(row["attributes"]) if row["attributes"] else {},
+                    "events": json.loads(row["events"]) if row["events"] else [],
+                    "agent_id": row["agent_id"],
+                    "project_id": row["project_id"],
+                    "classification": row["classification"],
+                }
+                for row in rows
+            ]
+        except sqlite3.Error as e:
+            logger.error("Query error: %s", e)
+            return []

tools/observability/trace_context.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python3
+# CUI // SP-CTI
+"""W3C Trace Context propagation (D281).
+
+Extends D149 correlation ID to W3C traceparent format for distributed tracing.
+Backward compatible — existing correlation IDs continue to work alongside traceparent.
+
+W3C traceparent format:
+    {version}-{trace_id}-{parent_id}-{flags}
+    00-4bf92f3577b58400000000000000000a-00f067aa0ba902b7-01
+
+Usage:
+    from tools.observability.trace_context import (
+        generate_traceparent,
+        parse_traceparent,
+        get_current_context,
+        set_current_context,
+    )
+
+    # Generate new trace context
+    ctx = generate_traceparent()  # -> TraceContext(trace_id, span_id, sampled)
+
+    # Parse incoming traceparent header
+    ctx = parse_traceparent("00-abc123...-def456...-01")
+
+    # Get/set in contextvars (propagates through async/thread boundaries)
+    set_current_context(ctx)
+    ctx = get_current_context()
+"""
+
+import contextvars
+import uuid
+from dataclasses import dataclass
+from typing import Optional
+
+
+# Version constant
+TRACEPARENT_VERSION = "00"
+
+# Context variable for propagation (works with threading and asyncio)
+_trace_context_var: contextvars.ContextVar[Optional["TraceContext"]] = contextvars.ContextVar(
+    "icdev_trace_context", default=None
+)
+
+
+@dataclass(frozen=True)
+class TraceContext:
+    """W3C Trace Context data (D281).
+
+    Attributes:
+        trace_id: 32-char hex string (128-bit trace identifier).
+        span_id: 16-char hex string (64-bit span identifier).
+        sampled: Whether this trace is sampled (included in export).
+        tracestate: Optional vendor-specific tracestate header value.
+    """
+    trace_id: str
+    span_id: str
+    sampled: bool = True
+    tracestate: str = ""
+
+    def to_traceparent(self) -> str:
+        """Format as W3C traceparent header value."""
+        flags = "01" if self.sampled else "00"
+        return f"{TRACEPARENT_VERSION}-{self.trace_id}-{self.span_id}-{flags}"
+
+    def to_correlation_id(self) -> str:
+        """Extract D149-compatible 12-char correlation ID from trace_id."""
+        return self.trace_id[:12]
+
+    def child(self, new_span_id: Optional[str] = None) -> "TraceContext":
+        """Create a child context with same trace_id but new span_id."""
+        return TraceContext(
+            trace_id=self.trace_id,
+            span_id=new_span_id or generate_span_id(),
+            sampled=self.sampled,
+            tracestate=self.tracestate,
+        )
+
+
+def generate_trace_id() -> str:
+    """Generate a 32-char hex trace ID (128-bit)."""
+    return uuid.uuid4().hex
+
+
+def generate_span_id() -> str:
+    """Generate a 16-char hex span ID (64-bit)."""
+    return uuid.uuid4().hex[:16]
+
+
+def generate_traceparent(
+    trace_id: Optional[str] = None,
+    span_id: Optional[str] = None,
+    sampled: bool = True,
+) -> TraceContext:
+    """Generate a new W3C trace context.
+
+    Args:
+        trace_id: Optional 32-char hex string. Generated if None.
+        span_id: Optional 16-char hex string. Generated if None.
+        sampled: Whether the trace is sampled. Default True.
+
+    Returns:
+        A new TraceContext.
+    """
+    return TraceContext(
+        trace_id=trace_id or generate_trace_id(),
+        span_id=span_id or generate_span_id(),
+        sampled=sampled,
+    )
+
+
+def parse_traceparent(header: str) -> Optional[TraceContext]:
+    """Parse a W3C traceparent header value.
+
+    Args:
+        header: traceparent string, e.g. "00-{trace_id}-{span_id}-{flags}"
+
+    Returns:
+        TraceContext if valid, None if malformed.
+    """
+    if not header:
+        return None
+
+    parts = header.strip().split("-")
+    if len(parts) != 4:
+        return None
+
+    version, trace_id, span_id, flags = parts
+
+    # Validate lengths
+    if len(trace_id) != 32 or len(span_id) != 16 or len(flags) != 2:
+        return None
+
+    # Validate hex
+    try:
+        int(trace_id, 16)
+        int(span_id, 16)
+        int(flags, 16)
+    except ValueError:
+        return None
+
+    # All-zero trace_id or span_id is invalid per spec
+    if trace_id == "0" * 32 or span_id == "0" * 16:
+        return None
+
+    sampled = (int(flags, 16) & 0x01) == 1
+    return TraceContext(trace_id=trace_id, span_id=span_id, sampled=sampled)
+
+
+def get_current_context() -> Optional[TraceContext]:
+    """Get the current trace context from contextvars.
+
+    Falls back to Flask g context if available (D149 integration).
+    """
+    ctx = _trace_context_var.get()
+    if ctx:
+        return ctx
+
+    # Try Flask g context for backward compat with D149
+    try:
+        from flask import g
+        traceparent = getattr(g, "traceparent", None)
+        if traceparent:
+            return parse_traceparent(traceparent)
+        # Fall back to correlation_id -> synthetic trace context
+        cid = getattr(g, "correlation_id", None)
+        if cid:
+            # Pad correlation_id to 32 chars for trace_id
+            padded_trace_id = (cid + "0" * 32)[:32]
+            return TraceContext(
+                trace_id=padded_trace_id,
+                span_id=generate_span_id(),
+                sampled=True,
+            )
+    except (ImportError, RuntimeError):
+        pass
+
+    return None
+
+
+def set_current_context(ctx: TraceContext) -> contextvars.Token:
+    """Set the current trace context in contextvars.
+
+    Returns a token that can be used to reset the context.
+    """
+    return _trace_context_var.set(ctx)
+
+
+def clear_current_context() -> None:
+    """Clear the current trace context."""
+    _trace_context_var.set(None)
+
+
+def context_from_correlation_id(correlation_id: str) -> TraceContext:
+    """Create a TraceContext from an existing D149 correlation ID.
+
+    Pads the 12-char correlation ID to a 32-char trace_id.
+    Used for backward compatibility during D149 → D281 transition.
+    """
+    padded = (correlation_id + "0" * 32)[:32]
+    return TraceContext(
+        trace_id=padded,
+        span_id=generate_span_id(),
+        sampled=True,
+    )