plain.observer 0.0.0__py3-none-any.whl

plain/observer/models.py

@@ -0,0 +1,355 @@
+import json
+from datetime import UTC, datetime
+from functools import cached_property
+
+import sqlparse
+from opentelemetry.semconv._incubating.attributes import (
+    exception_attributes,
+    session_attributes,
+    user_attributes,
+)
+from opentelemetry.semconv._incubating.attributes.db_attributes import (
+    DB_QUERY_PARAMETER_TEMPLATE,
+)
+from opentelemetry.semconv.attributes import db_attributes
+from opentelemetry.trace import format_trace_id
+
+from plain import models
+
+
+@models.register_model
+class Trace(models.Model):
+    trace_id = models.CharField(max_length=255)
+    start_time = models.DateTimeField()
+    end_time = models.DateTimeField()
+
+    root_span_name = models.TextField(default="", required=False)
+
+    # Plain fields
+    request_id = models.CharField(max_length=255, default="", required=False)
+    session_id = models.CharField(max_length=255, default="", required=False)
+    user_id = models.CharField(max_length=255, default="", required=False)
+
+    class Meta:
+        ordering = ["-start_time"]
+        constraints = [
+            models.UniqueConstraint(
+                fields=["trace_id"],
+                name="observer_unique_trace_id",
+            )
+        ]
+
+    def __str__(self):
+        return self.trace_id
+
+    def duration_ms(self):
+        return (self.end_time - self.start_time).total_seconds() * 1000
+
+    def get_trace_summary(self, spans=None):
+        """Get a concise summary string for toolbar display.
+
+        Args:
+            spans: Optional list of span objects. If not provided, will query from database.
+        """
+        # Get spans from database if not provided
+        if spans is None:
+            spans = list(self.spans.all())
+
+        if not spans:
+            return ""
+
+        # Count database queries and track duplicates
+        query_counts = {}
+        db_queries = 0
+
+        for span in spans:
+            if span.attributes.get(db_attributes.DB_SYSTEM_NAME):
+                db_queries += 1
+                if query_text := span.attributes.get(db_attributes.DB_QUERY_TEXT):
+                    query_counts[query_text] = query_counts.get(query_text, 0) + 1
+
+        # Count duplicate queries (queries that appear more than once)
+        duplicate_count = sum(count - 1 for count in query_counts.values() if count > 1)
+
+        # Build summary: "n spans, n queries (n duplicates), Xms"
+        parts = []
+
+        # Queries count with duplicates
+        if db_queries > 0:
+            query_part = f"{db_queries} quer{'y' if db_queries == 1 else 'ies'}"
+            if duplicate_count > 0:
+                query_part += f" ({duplicate_count} duplicate{'' if duplicate_count == 1 else 's'})"
+            parts.append(query_part)
+
+        # Duration
+        if (duration_ms := self.duration_ms()) is not None:
+            parts.append(f"{round(duration_ms, 1)}ms")
+
+        return " • ".join(parts)
+
+    @classmethod
+    def from_opentelemetry_spans(cls, spans):
+        """Create a Trace instance from a list of OpenTelemetry spans."""
+        # Get trace information from the first span
+        first_span = spans[0]
+        trace_id = f"0x{format_trace_id(first_span.get_span_context().trace_id)}"
+
+        # Find trace boundaries and root span info
+        earliest_start = None
+        latest_end = None
+        root_span = None
+        request_id = ""
+        user_id = ""
+        session_id = ""
+
+        for span in spans:
+            if not span.parent:
+                root_span = span
+
+            if span.start_time and (
+                earliest_start is None or span.start_time < earliest_start
+            ):
+                earliest_start = span.start_time
+            # Only update latest_end if the span has actually ended
+            if span.end_time and (latest_end is None or span.end_time > latest_end):
+                latest_end = span.end_time
+
+            # For OpenTelemetry spans, access attributes directly
+            span_attrs = getattr(span, "attributes", {})
+            request_id = request_id or span_attrs.get("plain.request.id", "")
+            user_id = user_id or span_attrs.get(user_attributes.USER_ID, "")
+            session_id = session_id or span_attrs.get(session_attributes.SESSION_ID, "")
+
+        # Convert timestamps
+        start_time = (
+            datetime.fromtimestamp(earliest_start / 1_000_000_000, tz=UTC)
+            if earliest_start
+            else None
+        )
+        end_time = (
+            datetime.fromtimestamp(latest_end / 1_000_000_000, tz=UTC)
+            if latest_end
+            else None
+        )
+
+        # Create trace instance
+        # Note: end_time might be None if there are active spans
+        # This is OK since this trace is only used for summaries, not persistence
+        return cls(
+            trace_id=trace_id,
+            start_time=start_time,
+            end_time=end_time
+            or start_time,  # Use start_time as fallback for active traces
+            request_id=request_id,
+            user_id=user_id,
+            session_id=session_id,
+            root_span_name=root_span.name if root_span else "",
+        )
+
+    def get_annotated_spans(self):
+        """Return spans with annotations and nesting information."""
+        spans = list(self.spans.all().order_by("start_time"))
+
+        # Build span dictionary for parent lookups
+        span_dict = {span.span_id: span for span in spans}
+
+        # Calculate nesting levels
+        for span in spans:
+            if not span.parent_id:
+                span.level = 0
+            else:
+                # Find parent's level and add 1
+                parent = span_dict.get(span.parent_id)
+                parent_level = parent.level if parent else 0
+                span.level = parent_level + 1
+
+        query_counts = {}
+
+        # First pass: count queries
+        for span in spans:
+            if sql_query := span.sql_query:
+                query_counts[sql_query] = query_counts.get(sql_query, 0) + 1
+
+        # Second pass: add annotations
+        query_occurrences = {}
+        for span in spans:
+            span.annotations = []
+
+            # Check for duplicate queries
+            if sql_query := span.sql_query:
+                count = query_counts[sql_query]
+                if count > 1:
+                    occurrence = query_occurrences.get(sql_query, 0) + 1
+                    query_occurrences[sql_query] = occurrence
+
+                    span.annotations.append(
+                        {
+                            "message": f"Duplicate query ({occurrence} of {count})",
+                            "severity": "warning",
+                        }
+                    )
+
+        return spans
+
+    def as_dict(self):
+        spans = [span.span_data for span in self.spans.all().order_by("start_time")]
+
+        return {
+            "trace_id": self.trace_id,
+            "start_time": self.start_time.isoformat(),
+            "end_time": self.end_time.isoformat(),
+            "duration_ms": self.duration_ms(),
+            "request_id": self.request_id,
+            "user_id": self.user_id,
+            "session_id": self.session_id,
+            "spans": spans,
+        }
+
+
+@models.register_model
+class Span(models.Model):
+    trace = models.ForeignKey(Trace, on_delete=models.CASCADE, related_name="spans")
+
+    span_id = models.CharField(max_length=255)
+
+    name = models.CharField(max_length=255)
+    kind = models.CharField(max_length=50)
+    parent_id = models.CharField(max_length=255, default="", required=False)
+    start_time = models.DateTimeField()
+    end_time = models.DateTimeField()
+    status = models.CharField(max_length=50, default="", required=False)
+    span_data = models.JSONField(default=dict, required=False)
+
+    class Meta:
+        ordering = ["-start_time"]
+        constraints = [
+            models.UniqueConstraint(
+                fields=["trace", "span_id"],
+                name="observer_unique_span_id",
+            )
+        ]
+        indexes = [
+            models.Index(fields=["trace", "span_id"]),
+            models.Index(fields=["trace"]),
+            models.Index(fields=["start_time"]),
+        ]
+
+    @classmethod
+    def from_opentelemetry_span(cls, otel_span, trace):
+        """Create a Span instance from an OpenTelemetry span."""
+
+        span_data = json.loads(otel_span.to_json())
+
+        # Extract status code as string, default to empty string if unset
+        status = ""
+        if span_data.get("status") and span_data["status"].get("status_code"):
+            status = span_data["status"]["status_code"]
+
+        return cls(
+            trace=trace,
+            span_id=span_data["context"]["span_id"],
+            name=span_data["name"],
+            kind=span_data["kind"][len("SpanKind.") :],
+            parent_id=span_data["parent_id"] or "",
+            start_time=span_data["start_time"],
+            end_time=span_data["end_time"],
+            status=status,
+            span_data=span_data,
+        )
+
+    def __str__(self):
+        return self.span_id
+
+    @property
+    def attributes(self):
+        """Get attributes from span_data."""
+        return self.span_data.get("attributes", {})
+
+    @property
+    def events(self):
+        """Get events from span_data."""
+        return self.span_data.get("events", [])
+
+    @property
+    def links(self):
+        """Get links from span_data."""
+        return self.span_data.get("links", [])
+
+    @property
+    def resource(self):
+        """Get resource from span_data."""
+        return self.span_data.get("resource", {})
+
+    @property
+    def context(self):
+        """Get context from span_data."""
+        return self.span_data.get("context", {})
+
+    def duration_ms(self):
+        if self.start_time and self.end_time:
+            return (self.end_time - self.start_time).total_seconds() * 1000
+        return 0
+
+    @cached_property
+    def sql_query(self):
+        """Get the SQL query if this span contains one."""
+        return self.attributes.get(db_attributes.DB_QUERY_TEXT)
+
+    @cached_property
+    def sql_query_params(self):
+        """Get query parameters from attributes that start with 'db.query.parameter.'"""
+        if not self.attributes:
+            return {}
+
+        query_params = {}
+        for key, value in self.attributes.items():
+            if key.startswith(DB_QUERY_PARAMETER_TEMPLATE + "."):
+                param_name = key.replace(DB_QUERY_PARAMETER_TEMPLATE + ".", "")
+                query_params[param_name] = value
+
+        return query_params
+
+    def get_formatted_sql(self):
+        """Get the pretty-formatted SQL query if this span contains one."""
+        sql = self.sql_query
+        if not sql:
+            return None
+
+        return sqlparse.format(
+            sql,
+            reindent=True,
+            keyword_case="upper",
+            identifier_case="lower",
+            strip_comments=False,
+            strip_whitespace=True,
+            indent_width=2,
+            wrap_after=80,
+            comma_first=False,
+        )
+
+    def format_event_timestamp(self, timestamp):
+        """Convert event timestamp to a readable datetime."""
+        if isinstance(timestamp, int | float):
+            try:
+                # Try as seconds first
+                if timestamp > 1e10:  # Likely nanoseconds
+                    timestamp = timestamp / 1e9
+                elif timestamp > 1e7:  # Likely milliseconds
+                    timestamp = timestamp / 1e3
+
+                return datetime.fromtimestamp(timestamp, tz=UTC)
+            except (ValueError, OSError):
+                return str(timestamp)
+        return timestamp
+
+    def get_exception_stacktrace(self):
+        """Get the exception stacktrace if this span has an exception event."""
+        if not self.events:
+            return None
+
+        for event in self.events:
+            if event.get("name") == "exception" and event.get("attributes"):
+                return event["attributes"].get(
+                    exception_attributes.EXCEPTION_STACKTRACE
+                )
+        return None

plain/observer/otel.py

@@ -0,0 +1,335 @@
+import logging
+import re
+import threading
+from collections import defaultdict
+
+import opentelemetry.context as context_api
+from opentelemetry import baggage, trace
+from opentelemetry.sdk.trace import SpanProcessor, sampling
+from opentelemetry.semconv.attributes import url_attributes
+from opentelemetry.trace import SpanKind, format_span_id, format_trace_id
+
+from plain.http.cookie import unsign_cookie_value
+from plain.models.otel import suppress_db_tracing
+from plain.runtime import settings
+
+from .core import Observer, ObserverMode
+
+logger = logging.getLogger(__name__)
+
+
+def get_span_processor():
+    """Get the span collector instance from the tracer provider."""
+    if not (current_provider := trace.get_tracer_provider()):
+        return None
+
+    # Look for ObserverSpanProcessor in the span processors
+    # Check if the provider has a _active_span_processor attribute
+    if hasattr(current_provider, "_active_span_processor"):
+        # It's a composite processor, check its _span_processors
+        if composite_processor := current_provider._active_span_processor:
+            if hasattr(composite_processor, "_span_processors"):
+                for processor in composite_processor._span_processors:
+                    if isinstance(processor, ObserverSpanProcessor):
+                        return processor
+
+    return None
+
+
+def get_current_trace_summary() -> str | None:
+    """Get performance summary for the currently active trace."""
+    if not (current_span := trace.get_current_span()):
+        return None
+
+    if not (processor := get_span_processor()):
+        return None
+
+    trace_id = f"0x{format_trace_id(current_span.get_span_context().trace_id)}"
+    return processor.get_trace_summary(trace_id)
+
+
+class ObserverSampler(sampling.Sampler):
+    """Samples traces based on request path and cookies."""
+
+    def __init__(self):
+        # Custom parent-based sampler
+        self._delegate = sampling.ParentBased(sampling.ALWAYS_OFF)
+
+        # TODO ignore url namespace instead? admin, observer, assets
+        self._ignore_url_paths = [
+            re.compile(p) for p in settings.OBSERVER_IGNORE_URL_PATTERNS
+        ]
+
+    def should_sample(
+        self,
+        parent_context,
+        trace_id,
+        name,
+        kind: SpanKind | None = None,
+        attributes=None,
+        links=None,
+        trace_state=None,
+    ):
+        # First, drop if the URL should be ignored.
+        if attributes:
+            if url_path := attributes.get(url_attributes.URL_PATH, ""):
+                for pattern in self._ignore_url_paths:
+                    if pattern.match(url_path):
+                        return sampling.SamplingResult(
+                            sampling.Decision.DROP,
+                            attributes=attributes,
+                        )
+
+        # If no processor decision, check cookies directly for root spans
+        decision = None
+        if parent_context:
+            # Check cookies for sampling decision
+            if cookies := baggage.get_baggage("http.request.cookies", parent_context):
+                if observer_cookie := cookies.get(Observer.COOKIE_NAME):
+                    unsigned_value = unsign_cookie_value(
+                        Observer.COOKIE_NAME, observer_cookie, default=False
+                    )
+
+                    if unsigned_value in (
+                        ObserverMode.PERSIST.value,
+                        ObserverMode.SUMMARY.value,
+                    ):
+                        # Always use RECORD_AND_SAMPLE so ParentBased works correctly
+                        # The processor will check the mode to decide whether to export
+                        decision = sampling.Decision.RECORD_AND_SAMPLE
+                    else:
+                        decision = sampling.Decision.DROP
+
+        # If there are links, assume it is to another trace/span that we are keeping
+        if links:
+            decision = sampling.Decision.RECORD_AND_SAMPLE
+
+        # If no decision from cookies, use default
+        if decision is None:
+            result = self._delegate.should_sample(
+                parent_context,
+                trace_id,
+                name,
+                kind=kind,
+                attributes=attributes,
+                links=links,
+                trace_state=trace_state,
+            )
+            decision = result.decision
+
+        return sampling.SamplingResult(
+            decision,
+            attributes=attributes,
+        )
+
+    def get_description(self) -> str:
+        return "ObserverSampler"
+
+
+class ObserverSpanProcessor(SpanProcessor):
+    """Collects spans in real-time for current trace performance monitoring.
+
+    This processor keeps spans in memory for traces that have the 'summary' or 'persist'
+    cookie set. These spans can be accessed via get_current_trace_summary() for
+    real-time debugging. Spans with 'persist' cookie will also be persisted to the
+    database.
+    """
+
+    def __init__(self):
+        # Span storage
+        self._traces = defaultdict(
+            lambda: {
+                "trace": None,  # Trace model instance
+                "active_otel_spans": {},  # span_id -> opentelemetry span
+                "completed_otel_spans": [],  # list of opentelemetry spans
+                "span_models": [],  # list of Span model instances
+                "root_span_id": None,
+                "mode": None,  # None, ObserverMode.SUMMARY.value, or ObserverMode.PERSIST.value
+            }
+        )
+        self._traces_lock = threading.Lock()
+
+    def on_start(self, span, parent_context=None):
+        """Called when a span starts."""
+        trace_id = f"0x{format_trace_id(span.get_span_context().trace_id)}"
+
+        with self._traces_lock:
+            # Check if we already have this trace
+            if trace_id in self._traces:
+                trace_info = self._traces[trace_id]
+            else:
+                # First span in trace - determine if we should record it
+                mode = self._get_recording_mode(span, parent_context)
+                if not mode:
+                    # Don't create trace entry for traces we won't record
+                    return
+
+                # Create trace entry only for traces we'll record
+                trace_info = self._traces[trace_id]
+                trace_info["mode"] = mode
+
+                # Clean up old traces if too many
+                if len(self._traces) > 1000:
+                    # Remove oldest 100 traces
+                    oldest_ids = sorted(self._traces.keys())[:100]
+                    for old_id in oldest_ids:
+                        del self._traces[old_id]
+
+            span_id = f"0x{format_span_id(span.get_span_context().span_id)}"
+
+            # Store span (we know mode is truthy if we get here)
+            trace_info["active_otel_spans"][span_id] = span
+
+            # Track root span
+            if not span.parent:
+                trace_info["root_span_id"] = span_id
+
+    def on_end(self, span):
+        """Called when a span ends."""
+        trace_id = f"0x{format_trace_id(span.get_span_context().trace_id)}"
+        span_id = f"0x{format_span_id(span.get_span_context().span_id)}"
+
+        with self._traces_lock:
+            # Skip if we don't have this trace (mode was None on start)
+            if trace_id not in self._traces:
+                return
+
+            trace_info = self._traces[trace_id]
+
+            # Move span from active to completed
+            if trace_info["active_otel_spans"].pop(span_id, None):
+                trace_info["completed_otel_spans"].append(span)
+
+            # Check if trace is complete (root span ended)
+            if span_id == trace_info["root_span_id"]:
+                all_spans = trace_info["completed_otel_spans"]
+
+                from .models import Span, Trace
+
+                trace_info["trace"] = Trace.from_opentelemetry_spans(all_spans)
+                trace_info["span_models"] = [
+                    Span.from_opentelemetry_span(s, trace_info["trace"])
+                    for s in all_spans
+                ]
+
+                # Export if in persist mode
+                if trace_info["mode"] == ObserverMode.PERSIST.value:
+                    logger.debug(
+                        "Exporting %d spans for trace %s",
+                        len(trace_info["span_models"]),
+                        trace_id,
+                    )
+                    self._export_trace(trace_info["trace"], trace_info["span_models"])
+
+                # Clean up trace
+                del self._traces[trace_id]
+
+    def get_trace_summary(self, trace_id: str) -> str | None:
+        """Get performance summary for a specific trace."""
+        from .models import Span, Trace
+
+        with self._traces_lock:
+            # Return None if trace doesn't exist (mode was None)
+            if trace_id not in self._traces:
+                return None
+
+            trace_info = self._traces[trace_id]
+
+            # Combine active and completed spans
+            all_otel_spans = (
+                list(trace_info["active_otel_spans"].values())
+                + trace_info["completed_otel_spans"]
+            )
+
+            if not all_otel_spans:
+                return None
+
+            # Create or update trace model instance
+            if not trace_info["trace"]:
+                trace_info["trace"] = Trace.from_opentelemetry_spans(all_otel_spans)
+
+            if not trace_info["trace"]:
+                return None
+
+            # Create span model instances if needed
+            span_models = trace_info.get("span_models", [])
+            if not span_models:
+                span_models = [
+                    Span.from_opentelemetry_span(s, trace_info["trace"])
+                    for s in all_otel_spans
+                ]
+
+            return trace_info["trace"].get_trace_summary(span_models)
+
+    def _export_trace(self, trace, span_models):
+        """Export trace and spans to the database."""
+        from .models import Span, Trace
+
+        with suppress_db_tracing():
+            try:
+                trace.save()
+
+                for span_model in span_models:
+                    span_model.trace = trace
+
+                # Bulk create spans
+                Span.objects.bulk_create(span_models)
+            except Exception as e:
+                logger.warning(
+                    "Failed to export trace to database: %s",
+                    e,
+                    exc_info=True,
+                )
+
+            # Delete oldest traces if we exceed the limit
+            if settings.OBSERVER_TRACE_LIMIT > 0:
+                try:
+                    if Trace.objects.count() > settings.OBSERVER_TRACE_LIMIT:
+                        delete_ids = Trace.objects.order_by("start_time")[
+                            : settings.OBSERVER_TRACE_LIMIT
+                        ].values_list("id", flat=True)
+                        Trace.objects.filter(id__in=delete_ids).delete()
+                except Exception as e:
+                    logger.warning(
+                        "Failed to clean up old observer traces: %s", e, exc_info=True
+                    )
+
+    def _get_recording_mode(self, span, parent_context) -> str | None:
+        # If the span has links, then we are going to export if the linked span is also exported
+        for link in span.links:
+            if link.context.is_valid and link.context.span_id:
+                from .models import Span
+
+                if Span.objects.filter(
+                    span_id=f"0x{format_span_id(link.context.span_id)}"
+                ).exists():
+                    return ObserverMode.PERSIST.value
+
+        if not (context := parent_context or context_api.get_current()):
+            return None
+
+        if not (cookies := baggage.get_baggage("http.request.cookies", context)):
+            return None
+
+        if not (observer_cookie := cookies.get(Observer.COOKIE_NAME)):
+            return None
+
+        try:
+            mode = unsign_cookie_value(
+                Observer.COOKIE_NAME, observer_cookie, default=None
+            )
+            if mode in (ObserverMode.SUMMARY.value, ObserverMode.PERSIST.value):
+                return mode
+        except Exception as e:
+            logger.warning("Failed to unsign observer cookie: %s", e)
+
+        return None
+
+    def shutdown(self):
+        """Cleanup when shutting down."""
+        with self._traces_lock:
+            self._traces.clear()
+
+    def force_flush(self, timeout_millis=None):
+        """Required by SpanProcessor interface."""
+        return True

plain/observer/templates/admin/observer/trace_detail.html

@@ -0,0 +1,10 @@
+{% extends "admin/detail.html" %}
+
+{% block content %}
+
+{{ super() }}
+
+<div class="mt-20">
+    {% include "observer/_trace_detail.html" %}
+</div>
+{% endblock %}