witdem-analytics 0.2.0__py3-none-any.whl

witdem/__init__.py

@@ -0,0 +1,3 @@
+"""Witdem runtime analytics product package."""
+
+__version__ = "0.2.0"

witdem/adapters/__init__.py

@@ -0,0 +1,13 @@
+"""Runtime adapter boundary: framework-specific span interpretation lives here.
+
+The analytics core (``analytics/core.py``, ``analytics/identity.py``,
+``analytics/runtime.py``) stays framework-neutral. Adapters translate a raw
+runtime's span/attribute conventions into the canonical
+``Execution``/``Operation``/``Link`` graph.
+"""
+
+from __future__ import annotations
+
+from witdem.adapters.base import RuntimeAdapter
+
+__all__ = ["RuntimeAdapter"]

witdem/adapters/base.py

@@ -0,0 +1,31 @@
+"""Public runtime adapter protocol used by built-in and external adapters."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from typing import Any, Protocol, runtime_checkable
+
+from witdem.analytics.runtime import NormalizedExecutionGraph
+
+
+@runtime_checkable
+class RuntimeAdapter(Protocol):
+    """Detects and normalizes one raw-runtime span convention.
+
+    ``spans`` are raw span envelopes in the same shape
+    ``telemetry.otel.JsonlSpanExporter`` produces (trace_id, span_id,
+    parent_span_id, name, kind, attributes, status, start_time_unix_nano,
+    end_time_unix_nano, events, resource, instrumentation_scope) —
+    ``ingest.otlp_http`` reconstructs this same shape from OTLP protobuf.
+    """
+
+    def detect(self, spans: Sequence[Mapping[str, Any]]) -> bool: ...
+
+    def normalize(
+        self,
+        spans: Sequence[Mapping[str, Any]],
+        *,
+        execution_id: str | None = None,
+        runtime_id: str | None = None,
+        providers: Sequence[Mapping[str, Any]] | None = None,
+    ) -> NormalizedExecutionGraph: ...

witdem/adapters/haystack_adapter.py

@@ -0,0 +1,59 @@
+"""Haystack/OpenTelemetry runtime adapter.
+
+Thin wrapper around the existing, already-tested
+``analytics.runtime.normalize_haystack_spans`` (see ``docs/architecture.md`` §3).
+All framework-specific *detection* knowledge (attribute/scope naming) lives
+here, isolated behind the ``RuntimeAdapter`` boundary; ``normalize()`` itself
+does not reimplement or modify the 650-line, already-tested normalizer it
+wraps.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+from witdem.analytics.runtime import NormalizedExecutionGraph, normalize_haystack_spans
+
+_HAYSTACK_ATTRIBUTE_PREFIX = "haystack."
+_HAYSTACK_SCOPE_HINTS = ("haystack", "opentelemetry-haystack")
+
+
+class HaystackAdapter:
+    """``RuntimeAdapter`` for Haystack-instrumented OpenTelemetry spans.
+
+    ``detect()`` looks for the same physical conventions
+    ``normalize_haystack_spans`` itself reads span-by-span: any
+    ``haystack.*``-prefixed attribute key (e.g. ``haystack.component.name``,
+    ``haystack.tool.name``, ``haystack.agent.step``), or an instrumentation
+    scope name that mentions "haystack" / "opentelemetry-haystack".
+    """
+
+    def detect(self, spans: Sequence[Mapping[str, Any]]) -> bool:
+        for span in spans:
+            attributes = span.get("attributes")
+            if isinstance(attributes, Mapping) and any(
+                str(key).startswith(_HAYSTACK_ATTRIBUTE_PREFIX) for key in attributes
+            ):
+                return True
+            scope = span.get("instrumentation_scope")
+            if isinstance(scope, Mapping):
+                name = str(scope.get("name") or "").casefold()
+                if any(hint in name for hint in _HAYSTACK_SCOPE_HINTS):
+                    return True
+        return False
+
+    def normalize(
+        self,
+        spans: Sequence[Mapping[str, Any]],
+        *,
+        execution_id: str | None = None,
+        runtime_id: str | None = None,
+        providers: Sequence[Mapping[str, Any]] | None = None,
+    ) -> NormalizedExecutionGraph:
+        return normalize_haystack_spans(
+            spans,
+            execution_id=execution_id,
+            runtime_id=runtime_id,
+            providers=providers,
+        )

witdem/adapters/providers.py

@@ -0,0 +1,110 @@
+"""Provider normalization executed after landing and before runtime adaptation."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+from witdem import __version__
+
+_PROVIDER_ALIASES = {
+    "openai": "openai",
+    "azure_openai": "azure_openai",
+    "azure.openai": "azure_openai",
+    "azure": "azure_openai",
+    "anthropic": "anthropic",
+    "claude": "anthropic",
+    "deepseek": "deepseek",
+    "mistral": "mistral",
+    "mistralai": "mistral",
+    "amazon_bedrock": "amazon_bedrock",
+    "aws.bedrock": "amazon_bedrock",
+    "bedrock": "amazon_bedrock",
+    "google": "google",
+    "google.vertex": "google",
+    "vertex": "google",
+    "vertex_ai": "google",
+    "gemini": "google",
+    "cohere": "cohere",
+    "ollama": "ollama",
+}
+
+_MODEL_PREFIXES = {
+    "gpt-": "openai",
+    "o1": "openai",
+    "o3": "openai",
+    "o4": "openai",
+    "claude-": "anthropic",
+    "deepseek-": "deepseek",
+    "mistral-": "mistral",
+    "ministral-": "mistral",
+    "codestral-": "mistral",
+}
+
+
+def _first(attributes: Mapping[str, Any], *keys: str) -> Any:
+    for key in keys:
+        value = attributes.get(key)
+        if value is not None and str(value):
+            return value
+    return None
+
+
+def _provider(attributes: Mapping[str, Any]) -> tuple[str | None, str | None]:
+    observed = _first(attributes, "gen_ai.provider.name", "gen_ai.system", "provider", "llm.provider")
+    if observed is not None:
+        canonical = _PROVIDER_ALIASES.get(str(observed).strip().casefold().replace("-", "_"))
+        return canonical or str(observed).strip().casefold(), "observed_attribute"
+    model = _first(attributes, "gen_ai.response.model", "gen_ai.request.model", "model", "llm.model_name")
+    lowered = str(model or "").casefold()
+    for prefix, provider in _MODEL_PREFIXES.items():
+        if lowered.startswith(prefix):
+            return provider, "model_prefix"
+    return None, None
+
+
+def normalize_provider_spans(
+    spans: Sequence[Mapping[str, Any]],
+) -> tuple[list[dict[str, Any]], tuple[str, ...]]:
+    """Normalize provider evidence per span without choosing one run-wide provider."""
+
+    normalized: list[dict[str, Any]] = []
+    adapters: set[str] = set()
+    for raw in spans:
+        row = dict(raw)
+        attributes_value = row.get("attributes")
+        attributes = dict(attributes_value) if isinstance(attributes_value, Mapping) else {}
+        provider, source = _provider(attributes)
+        if provider is not None:
+            adapters.add(provider)
+            observed_provider = _first(
+                attributes,
+                "gen_ai.provider.name",
+                "gen_ai.system",
+                "provider",
+                "llm.provider",
+            )
+            if observed_provider is not None and str(observed_provider).strip().casefold() != provider:
+                attributes["witdem.provider_adapter.observed"] = str(observed_provider)
+            attributes["provider"] = provider
+            attributes["gen_ai.provider.name"] = provider
+            attributes["witdem.provider_adapter.name"] = provider
+            attributes["witdem.provider_adapter.version"] = __version__
+            attributes["witdem.provider_adapter.source"] = source
+            model = _first(
+                attributes,
+                "gen_ai.response.model",
+                "gen_ai.request.model",
+                "model",
+                "llm.model_name",
+            )
+            gen_ai_operation = str(attributes.get("gen_ai.operation.name") or "").casefold()
+            has_usage = any(str(key).startswith("gen_ai.usage.") for key in attributes)
+            if model is not None and (gen_ai_operation or has_usage):
+                # Framework spans often use a generic name such as "Component".
+                # The provider adapter has stronger evidence than that name and
+                # records the canonical kind for the runtime adapter to consume.
+                attributes["witdem.operation.kind"] = "model"
+        row["attributes"] = attributes
+        normalized.append(row)
+    return normalized, tuple(sorted(adapters))

witdem/adapters/registry.py

@@ -0,0 +1,51 @@
+"""Runtime adapter registry (see ``docs/architecture.md`` §3).
+
+Used by the OTLP ingestion path (``ingest.correlate``) to select the first
+matching runtime-specific adapter.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+from witdem.adapters.base import RuntimeAdapter
+from witdem.adapters.haystack_adapter import HaystackAdapter
+from witdem.integrations.adapters.claude import ClaudeAdapter
+from witdem.integrations.adapters.langchain import LangChainAdapter
+from witdem.integrations.adapters.langgraph import LangGraphAdapter
+from witdem.integrations.adapters.openai_agents import OpenAIAgentsAdapter
+from witdem.integrations.adapters.otel import OTelAdapter
+
+logger = logging.getLogger(__name__)
+
+_REGISTERED_ADAPTERS: tuple[RuntimeAdapter, ...] = (
+    LangGraphAdapter(),
+    OpenAIAgentsAdapter(),
+    ClaudeAdapter(),
+    LangChainAdapter(),
+    HaystackAdapter(),
+    OTelAdapter(),
+)
+
+_FALLBACK_ADAPTER: RuntimeAdapter = OTelAdapter()
+
+
+def detect_adapter(spans: Sequence[Mapping[str, Any]]) -> RuntimeAdapter:
+    """Return the first registered adapter whose ``detect()`` is True.
+
+    Falls back to :class:`OTelAdapter` (with a debug log noting the
+    fallback) when no registered adapter positively matches -- see
+    ``docs/architecture.md`` §3 and the module docstring above.
+    """
+
+    for adapter in _REGISTERED_ADAPTERS:
+        if adapter.detect(spans):
+            return adapter
+    logger.debug(
+        "detect_adapter: no adapter positively matched %d spans; falling back to %s",
+        len(spans),
+        type(_FALLBACK_ADAPTER).__name__,
+    )
+    return _FALLBACK_ADAPTER

witdem/analytics/__init__.py

@@ -0,0 +1,74 @@
+"""Public, domain-neutral Witdem analytics API."""
+
+from witdem.analytics.contracts import (
+    CostSummary,
+    ExecutionSummary,
+    FailureSummary,
+    ModelSummary,
+    PathSummary,
+    PerformanceSummary,
+    ProviderSummary,
+)
+from witdem.analytics.core import Evaluation, Event, Execution, Link, Operation, Outcome
+from witdem.analytics.derived import derived_termination_category
+from witdem.analytics.identity import (
+    canonical_operation_key,
+    canonical_path_signature,
+    canonical_stage_key,
+    canonical_tool_key,
+    display_operation,
+    display_path,
+    display_stage,
+    display_tool,
+)
+from witdem.analytics.runtime import (
+    NormalizedExecutionGraph,
+    ReplayGraph,
+    derive_replay_graph,
+    derive_runtime_insights,
+    find_similar_executions,
+    normalize_haystack_spans,
+)
+from witdem.analytics.schema import (
+    AGGREGATE_COLUMNS,
+    ANALYTICS_COLUMN_TYPES,
+    ANALYTICS_COLUMNS,
+    ANALYTICS_TABLES,
+    V2_ANALYTICS_TABLES,
+)
+
+__all__ = [
+    "Event",
+    "Execution",
+    "Evaluation",
+    "Link",
+    "Operation",
+    "Outcome",
+    "ExecutionSummary",
+    "CostSummary",
+    "ProviderSummary",
+    "ModelSummary",
+    "FailureSummary",
+    "PerformanceSummary",
+    "PathSummary",
+    "NormalizedExecutionGraph",
+    "ReplayGraph",
+    "normalize_haystack_spans",
+    "derive_replay_graph",
+    "derive_runtime_insights",
+    "find_similar_executions",
+    "derived_termination_category",
+    "canonical_operation_key",
+    "canonical_path_signature",
+    "canonical_stage_key",
+    "canonical_tool_key",
+    "display_operation",
+    "display_path",
+    "display_stage",
+    "display_tool",
+    "ANALYTICS_TABLES",
+    "ANALYTICS_COLUMNS",
+    "ANALYTICS_COLUMN_TYPES",
+    "AGGREGATE_COLUMNS",
+    "V2_ANALYTICS_TABLES",
+]

witdem/analytics/contracts/__init__.py

@@ -0,0 +1,29 @@
+"""Stable, frontend-neutral analytics read models."""
+
+from witdem.analytics.contracts.read_models import (
+    CostSummary,
+    ExecutionSummary,
+    FailureSummary,
+    MetadataSnapshot,
+    ModelSummary,
+    OverviewSnapshot,
+    PathSummary,
+    PerformanceSummary,
+    ProductGoalSummary,
+    ProviderSummary,
+    SemanticReplayRecord,
+)
+
+__all__ = [
+    "CostSummary",
+    "ExecutionSummary",
+    "FailureSummary",
+    "MetadataSnapshot",
+    "ModelSummary",
+    "OverviewSnapshot",
+    "PathSummary",
+    "PerformanceSummary",
+    "ProductGoalSummary",
+    "ProviderSummary",
+    "SemanticReplayRecord",
+]

witdem/analytics/contracts/read_models.py

@@ -0,0 +1,199 @@
+"""Typed analytics results shared by UI, API, and future frontend clients."""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from datetime import datetime
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from witdem.analytics.repository.state import Capabilities
+
+
+@dataclass(frozen=True, slots=True)
+class _ReadModel:
+    """Common serialization boundary for frontend/API adapters."""
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass(frozen=True, slots=True)
+class SemanticReplayRecord(_ReadModel):
+    record_id: str
+    kind: str
+    name: str
+    timestamp: datetime | None
+    status: str | None
+    value: Any
+    attributes: dict[str, Any]
+
+
+@dataclass(frozen=True, slots=True)
+class ExecutionSummary(_ReadModel):
+    total_runs: int
+    successful_runs: int
+    failed_runs: int
+    running_runs: int
+    recovered_runs: int
+    extra_work_runs: int
+    avg_duration_seconds: float | None
+    measured_cost: float | None
+    cost_coverage: float
+    business_successful_runs: int
+    business_unsuccessful_runs: int
+    business_reported_runs: int
+
+
+@dataclass(frozen=True, slots=True)
+class ProductGoalSummary(_ReadModel):
+    total_runs: int
+    reported_runs: int
+    achieved_runs: int
+    decision_correct_runs: int
+    false_acceptances: int
+    false_rejections: int
+    escalation_errors: int
+    targeted_research_runs: int
+    targeted_research_successes: int
+    cost_per_achieved_goal: float | None
+    cost_measured_achieved_runs: int
+    time_per_achieved_goal: float | None
+    time_measured_achieved_runs: int
+    tokens_per_achieved_goal: float | None
+    token_measured_achieved_runs: int
+
+    @property
+    def cost_coverage(self) -> float:
+        return self.cost_measured_achieved_runs / self.achieved_runs if self.achieved_runs else 0.0
+
+    @property
+    def time_coverage(self) -> float:
+        return self.time_measured_achieved_runs / self.achieved_runs if self.achieved_runs else 0.0
+
+    @property
+    def token_coverage(self) -> float:
+        return self.token_measured_achieved_runs / self.achieved_runs if self.achieved_runs else 0.0
+
+    @property
+    def coverage(self) -> float:
+        return self.reported_runs / self.total_runs if self.total_runs else 0.0
+
+    @property
+    def success_rate(self) -> float:
+        return self.achieved_runs / self.reported_runs if self.reported_runs else 0.0
+
+    @property
+    def decision_correctness_rate(self) -> float:
+        return self.decision_correct_runs / self.reported_runs if self.reported_runs else 0.0
+
+
+@dataclass(frozen=True, slots=True)
+class CostSummary(_ReadModel):
+    measured_cost: float | None
+    model_cost: float | None
+    tool_cost: float | None
+    cost_coverage: float
+    measured_cost_per_run: float | None
+    input_tokens: float | None
+    output_tokens: float | None
+    total_tokens: float | None
+    token_runs: int
+
+
+@dataclass(frozen=True, slots=True)
+class OverviewSnapshot(_ReadModel):
+    """One coherent read model for the dashboard overview request."""
+
+    execution: ExecutionSummary
+    goals: ProductGoalSummary
+    costs: CostSummary
+    cost_unavailable: dict[str, int]
+    models: tuple[ModelSummary, ...]
+    providers: tuple[ProviderSummary, ...]
+    workflows: tuple[PerformanceSummary, ...]
+    stages: tuple[dict[str, Any], ...]
+    runtime_breakdown: dict[str, int]
+    outcome_breakdown: dict[str, int]
+    failures: tuple[FailureSummary, ...]
+    evaluations: tuple[dict[str, Any], ...]
+    goal_misses: tuple[dict[str, Any], ...]
+    goal_trend: tuple[dict[str, Any], ...]
+    goal_portfolio: tuple[dict[str, Any], ...]
+    assurance_summary: dict[str, int | float]
+    contracts: tuple[dict[str, Any], ...]
+    metadata: MetadataSnapshot
+
+
+@dataclass(frozen=True, slots=True)
+class MetadataSnapshot(_ReadModel):
+    """One coherent read model for dashboard capabilities and filter values."""
+
+    capabilities: Capabilities
+    filters: dict[str, tuple[str, ...]]
+    contracts: tuple[dict[str, Any], ...]
+
+
+@dataclass(frozen=True, slots=True)
+class PerformanceSummary(_ReadModel):
+    label: str
+    runs: int
+    calls: int
+    completed: int
+    successful: int
+    failed: int
+    recovered: int
+    extra_work: int
+    measured_cost: float | None
+    cost_per_positive_run: float | None
+    time_per_positive_run: float | None
+    failed_run_cost: float | None
+    total_tokens: float | None
+    tokens_per_positive_run: float | None
+    failed_run_tokens: float | None
+    failure_rate: float
+    extra_work_rate: float
+    cost_coverage: float
+    semantics: str
+
+
+@dataclass(frozen=True, slots=True)
+class ProviderSummary(PerformanceSummary):
+    """Performance summary grouped by provider."""
+
+
+@dataclass(frozen=True, slots=True)
+class ModelSummary(PerformanceSummary):
+    """Performance summary grouped by model."""
+
+
+@dataclass(frozen=True, slots=True)
+class FailureSummary(_ReadModel):
+    failure_location: str
+    failure_key: str
+    kind: str
+    failures: int
+    executions: int
+    terminal_runs: int
+    recovered_runs: int
+    unknown_outcome_runs: int
+    providers: str | None
+    models: str | None
+    time_seconds: float
+    known_cost: float | None
+    total_tokens: float | None
+
+
+@dataclass(frozen=True, slots=True)
+class PathSummary(_ReadModel):
+    path: str
+    steps: tuple[str, ...]
+    path_signature: str
+    executions: int
+    completed: int
+    failures: int
+    failure_reports: int
+    time_seconds: float
+    usual_seconds: float | None
+    known_cost: float | None
+    total_tokens: float | None

witdem/analytics/core.py

@@ -0,0 +1,108 @@
+"""Small domain-neutral analytics records.
+
+This module deliberately knows nothing about any business domain, framework, or
+provider. Domain vocabulary is carried by strings and validated by a domain
+layer before records are persisted.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+from uuid import uuid4
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+def utc_now() -> datetime:
+    """Return a timezone-aware timezone.utc timestamp."""
+
+    return datetime.now(timezone.utc)
+
+
+class AnalyticsRecord(BaseModel):
+    """Strict base for persisted analytics records."""
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class Execution(AnalyticsRecord):
+    """One unit of workflow or runtime work."""
+
+    execution_id: str
+    runtime_id: str | None = None
+    started_at: datetime | None = None
+    ended_at: datetime | None = None
+    status: str | None = None
+    schema_version: str = "0.1.0"
+    attributes: dict[str, Any] = Field(default_factory=dict)
+
+
+class Operation(AnalyticsRecord):
+    """Work performed within an execution."""
+
+    operation_id: str = Field(default_factory=lambda: uuid4().hex)
+    execution_id: str
+    trace_id: str | None = None
+    span_id: str | None = None
+    parent_span_id: str | None = None
+    kind: str
+    name: str
+    status: str | None = None
+    started_at: datetime | None = None
+    ended_at: datetime | None = None
+    attempt: int | None = Field(default=None, ge=1)
+    attributes: dict[str, Any] = Field(default_factory=dict)
+
+
+class Event(AnalyticsRecord):
+    """A meaningful instantaneous occurrence."""
+
+    event_id: str = Field(default_factory=lambda: uuid4().hex)
+    execution_id: str
+    trace_id: str | None = None
+    span_id: str | None = None
+    timestamp: datetime = Field(default_factory=utc_now)
+    type: str = Field(min_length=1)
+    name: str = Field(min_length=1)
+    payload: dict[str, Any] = Field(default_factory=dict)
+    schema_version: str = "0.1.0"
+
+
+class Link(AnalyticsRecord):
+    """A relationship not represented by operation parentage alone."""
+
+    link_id: str = Field(default_factory=lambda: uuid4().hex)
+    execution_id: str
+    source_id: str
+    target_id: str
+    relation: str = Field(min_length=1)
+    attributes: dict[str, Any] = Field(default_factory=dict)
+
+
+class Evaluation(AnalyticsRecord):
+    """A structured assessment of an execution, operation, result, or object."""
+
+    evaluation_id: str = Field(default_factory=lambda: uuid4().hex)
+    execution_id: str
+    subject_id: str | None = None
+    name: str = Field(min_length=1)
+    value: Any = None
+    label: str | None = None
+    score: float | None = Field(default=None, ge=0, le=1)
+    source: str = Field(min_length=1)
+    confidence: float | None = Field(default=None, ge=0, le=1)
+    definition_version: str | None = None
+    attributes: dict[str, Any] = Field(default_factory=dict)
+
+
+class Outcome(AnalyticsRecord):
+    """An externally meaningful result of an execution."""
+
+    outcome_id: str = Field(default_factory=lambda: uuid4().hex)
+    execution_id: str
+    name: str = Field(min_length=1)
+    status: str | None = None
+    value: Any = None
+    timestamp: datetime = Field(default_factory=utc_now)
+    attributes: dict[str, Any] = Field(default_factory=dict)