evalgate-sdk 3.3.1__py3-none-any.whl

evalgate_sdk/cluster.py

@@ -0,0 +1,359 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from evalgate_sdk.golden import NormalizedRunArtifact, NormalizedRunCase
+
+STOP_WORDS = {
+    "the",
+    "and",
+    "for",
+    "with",
+    "that",
+    "this",
+    "from",
+    "into",
+    "your",
+    "have",
+    "should",
+    "would",
+    "could",
+    "about",
+    "what",
+    "when",
+    "where",
+    "while",
+    "were",
+    "them",
+    "then",
+    "than",
+    "also",
+    "been",
+    "because",
+    "expected",
+    "actual",
+    "input",
+    "output",
+    "error",
+    "failed",
+    "passed",
+    "skipped",
+    "result",
+    "spec",
+    "case",
+    "file",
+}
+
+
+@dataclass(slots=True)
+class ClusterSample:
+    case_id: str
+    name: str
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "caseId": self.case_id,
+            "name": self.name,
+        }
+
+
+@dataclass(slots=True)
+class ClusterCase:
+    case_id: str
+    name: str
+    file_path: str
+    status: str
+    input: str
+    expected: str
+    actual: str
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "caseId": self.case_id,
+            "name": self.name,
+            "filePath": self.file_path,
+            "status": self.status,
+            "input": self.input,
+            "expected": self.expected,
+            "actual": self.actual,
+        }
+
+
+@dataclass(slots=True)
+class TraceCluster:
+    id: str
+    cluster_label: str
+    dominant_pattern: str
+    suggested_failure_mode: str | None
+    similarity_threshold: float
+    trace_ids: list[str]
+    trace_count: int
+    keywords: list[str]
+    member_ids: list[str]
+    member_count: int
+    density: float
+    status_counts: dict[str, int]
+    samples: list[ClusterSample] = field(default_factory=list)
+    cases: list[ClusterCase] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "clusterLabel": self.cluster_label,
+            "dominantPattern": self.dominant_pattern,
+            "suggestedFailureMode": self.suggested_failure_mode,
+            "similarityThreshold": self.similarity_threshold,
+            "traceIds": list(self.trace_ids),
+            "traceCount": self.trace_count,
+            "keywords": list(self.keywords),
+            "memberIds": list(self.member_ids),
+            "memberCount": self.member_count,
+            "density": self.density,
+            "statusCounts": dict(self.status_counts),
+            "samples": [sample.to_dict() for sample in self.samples],
+            "cases": [case.to_dict() for case in self.cases],
+        }
+
+
+@dataclass(slots=True)
+class ClusterSummary:
+    run_id: str
+    total_run_results: int
+    clustered_cases: int
+    skipped_cases: int
+    requested_clusters: int | None
+    include_passed: bool
+    clusters: list[TraceCluster]
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "runId": self.run_id,
+            "totalRunResults": self.total_run_results,
+            "clusteredCases": self.clustered_cases,
+            "skippedCases": self.skipped_cases,
+            "requestedClusters": self.requested_clusters,
+            "includePassed": self.include_passed,
+            "clusters": [cluster.to_dict() for cluster in self.clusters],
+        }
+
+
+def _tokenize(text: str) -> list[str]:
+    normalized = []
+    current = []
+    for char in text.lower():
+        if char.isalnum():
+            current.append(char)
+        else:
+            if current:
+                token = "".join(current)
+                if len(token) > 2 and token not in STOP_WORDS:
+                    normalized.append(token)
+                current = []
+    if current:
+        token = "".join(current)
+        if len(token) > 2 and token not in STOP_WORDS:
+            normalized.append(token)
+    return normalized
+
+
+def _token_set(text: str) -> set[str]:
+    return set(_tokenize(text))
+
+
+def _jaccard(a: set[str], b: set[str]) -> float:
+    if not a and not b:
+        return 1.0
+    if not a or not b:
+        return 0.0
+    intersection = len(a.intersection(b))
+    union = len(a.union(b))
+    return intersection / union if union else 0.0
+
+
+def _build_case_text(case: NormalizedRunCase) -> str:
+    return "\n".join(
+        part
+        for part in [case.name, case.file_path, case.error or "", case.input, case.expected, case.actual]
+        if part.strip()
+    )
+
+
+def _centroid_keywords(texts: list[str], top_n: int = 4) -> list[str]:
+    frequencies: dict[str, int] = {}
+    for text in texts:
+        for token in _tokenize(text):
+            frequencies[token] = frequencies.get(token, 0) + 1
+    ordered = sorted(frequencies.items(), key=lambda item: (-item[1], item[0]))
+    return [token for token, _count in ordered[:top_n]]
+
+
+def _suggest_failure_mode(text: str, has_failed_members: bool) -> str | None:
+    if not has_failed_members:
+        return None
+    lower = text.lower()
+    if "timeout" in lower or "slow" in lower:
+        return "performance_timeout"
+    if "null" in lower or "undefined" in lower:
+        return "null_reference"
+    if "format" in lower or "parse" in lower:
+        return "format_mismatch"
+    if "constraint" in lower or "validation" in lower:
+        return "constraint_violation"
+    if "tone" in lower or "empathetic" in lower or "professional" in lower:
+        return "tone_mismatch"
+    if "halluc" in lower or "invented" in lower or "grounding" in lower:
+        return "hallucination"
+    return "general_failure"
+
+
+def _density(member_sets: list[set[str]]) -> float:
+    if len(member_sets) < 2:
+        return 1.0
+    total = 0.0
+    count = 0
+    for index, current in enumerate(member_sets):
+        for other in member_sets[index + 1 :]:
+            total += _jaccard(current, other)
+            count += 1
+    return total / count if count else 1.0
+
+
+def _assign_clusters(points: list[dict[str, Any]], cluster_count: int) -> list[list[dict[str, Any]]]:
+    if not points:
+        return []
+    clusters: list[list[dict[str, Any]]] = []
+    for point in points:
+        if len(clusters) < cluster_count:
+            clusters.append([point])
+            continue
+        best_index = 0
+        best_score = -1.0
+        for index, cluster in enumerate(clusters):
+            scores = [_jaccard(point["tokens"], member["tokens"]) for member in cluster]
+            avg_score = sum(scores) / len(scores) if scores else 0.0
+            if avg_score > best_score:
+                best_score = avg_score
+                best_index = index
+        clusters[best_index].append(point)
+    return clusters
+
+
+def cluster_run_result(
+    run_artifact: NormalizedRunArtifact,
+    *,
+    clusters: int | None = None,
+    include_passed: bool = False,
+) -> ClusterSummary:
+    candidates = [case for case in run_artifact.cases if include_passed or case.status == "failed"]
+    if not candidates:
+        return ClusterSummary(
+            run_id=run_artifact.run_id,
+            total_run_results=run_artifact.total_run_results,
+            clustered_cases=0,
+            skipped_cases=run_artifact.total_run_results,
+            requested_clusters=clusters,
+            include_passed=include_passed,
+            clusters=[],
+        )
+
+    points = []
+    for case in candidates:
+        text = _build_case_text(case)
+        points.append({"case": case, "text": text, "tokens": _token_set(text)})
+
+    cluster_count = clusters or min(8, max(1, round(len(points) ** 0.5)))
+    grouped = _assign_clusters(points, min(cluster_count, len(points)))
+
+    built_clusters: list[TraceCluster] = []
+    for index, members in enumerate(grouped):
+        texts = [member["text"] for member in members]
+        keywords = _centroid_keywords(texts)
+        dominant_pattern = ", ".join(keywords[:3]) if keywords else "No clear pattern"
+        failed_text = " ".join(member["text"] for member in members if member["case"].status == "failed")
+        status_counts = {"passed": 0, "failed": 0, "skipped": 0}
+        for member in members:
+            status = member["case"].status
+            if status == "passed":
+                status_counts["passed"] += 1
+            elif status == "skipped":
+                status_counts["skipped"] += 1
+            else:
+                status_counts["failed"] += 1
+        member_token_sets = [member["tokens"] for member in members]
+        density = _density(member_token_sets)
+        cases = [
+            ClusterCase(
+                case_id=member["case"].case_id,
+                name=member["case"].name,
+                file_path=member["case"].file_path,
+                status=member["case"].status,
+                input=member["case"].input,
+                expected=member["case"].expected,
+                actual=member["case"].actual,
+            )
+            for member in members
+        ]
+        built_clusters.append(
+            TraceCluster(
+                id=f"cluster-{index}",
+                cluster_label=", ".join(keywords[:3]) if keywords else f"Cluster {index + 1}",
+                dominant_pattern=dominant_pattern,
+                suggested_failure_mode=_suggest_failure_mode(failed_text, status_counts["failed"] > 0),
+                similarity_threshold=density,
+                trace_ids=[member["case"].case_id for member in members],
+                trace_count=len(members),
+                keywords=keywords,
+                member_ids=[member["case"].case_id for member in members],
+                member_count=len(members),
+                density=density,
+                status_counts=status_counts,
+                samples=[
+                    ClusterSample(case_id=member["case"].case_id, name=member["case"].name)
+                    for member in members[:3]
+                ],
+                cases=cases,
+            )
+        )
+
+    built_clusters.sort(key=lambda item: (-item.trace_count, -item.similarity_threshold, item.id))
+    return ClusterSummary(
+        run_id=run_artifact.run_id,
+        total_run_results=run_artifact.total_run_results,
+        clustered_cases=len(candidates),
+        skipped_cases=run_artifact.total_run_results - len(candidates),
+        requested_clusters=clusters,
+        include_passed=include_passed,
+        clusters=built_clusters,
+    )
+
+
+def format_cluster_human(summary: ClusterSummary) -> str:
+    lines = [
+        "Cluster phase",
+        f"Run: {summary.run_id}",
+        f"Clustered {summary.clustered_cases} case(s) into {len(summary.clusters)} cluster(s)",
+    ]
+    if summary.skipped_cases > 0:
+        lines.append(
+            f"Skipped {summary.skipped_cases} case(s) ({'none filtered' if summary.include_passed else 'use --include-passed to include non-failures'})"
+        )
+    if not summary.clusters:
+        lines.append("No cases available for clustering")
+        return "\n".join(lines)
+    for index, cluster in enumerate(summary.clusters, start=1):
+        lines.append("")
+        lines.append(
+            f"{index}. {cluster.id} — {cluster.cluster_label} ({cluster.trace_count} case(s), {(cluster.similarity_threshold * 100):.1f}% similarity)"
+        )
+        lines.append(
+            f"   status: {cluster.status_counts['failed']} failed, {cluster.status_counts['passed']} passed, {cluster.status_counts['skipped']} skipped"
+        )
+        if cluster.dominant_pattern:
+            lines.append(f"   pattern: {cluster.dominant_pattern}")
+        if cluster.suggested_failure_mode:
+            lines.append(f"   suggested failure mode: {cluster.suggested_failure_mode}")
+        if cluster.samples:
+            sample_line = ", ".join(f"{sample.case_id} ({sample.name})" for sample in cluster.samples)
+            lines.append(f"   samples: {sample_line}")
+    return "\n".join(lines)

evalgate_sdk/collector.py

@@ -0,0 +1,161 @@
+"""Production trace collector — report traces with client-side sampling.
+
+Port of the TypeScript SDK's ``collector.ts``.
+
+Usage::
+
+    from evalgate_sdk import AIEvalClient
+    from evalgate_sdk.collector import report_trace
+
+    client = AIEvalClient(api_key="...")
+    result = await report_trace(client, ReportTraceInput(
+        trace_id="t-123",
+        name="chat-completion",
+        spans=[CollectorSpanInput(span_id="s-1", name="llm-call")],
+    ))
+"""
+
+from __future__ import annotations
+
+import random
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+
+@dataclass
+class CollectorSpanInput:
+    span_id: str
+    name: str
+    type: Literal["llm", "tool", "agent", "retrieval", "default"] | None = None
+    parent_span_id: str | None = None
+    input: Any = None
+    output: Any = None
+    model: str | None = None
+    vendor: str | None = None
+    params: dict[str, Any] | None = None
+    metrics: dict[str, Any] | None = None
+    timestamps: dict[str, float] | None = None
+    error: dict[str, Any] | None = None
+    behavioral: dict[str, Any] | None = None
+    metadata: dict[str, Any] | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        d: dict[str, Any] = {"span_id": self.span_id, "name": self.name}
+        for key in (
+            "type",
+            "parent_span_id",
+            "input",
+            "output",
+            "model",
+            "vendor",
+            "params",
+            "metrics",
+            "timestamps",
+            "error",
+            "behavioral",
+            "metadata",
+        ):
+            val = getattr(self, key)
+            if val is not None:
+                d[key] = val
+        return d
+
+
+@dataclass
+class CollectorFeedbackInput:
+    type: Literal["thumbs_up", "thumbs_down", "rating", "comment"]
+    value: Any = None
+    user_id: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        d: dict[str, Any] = {"type": self.type}
+        if self.value is not None:
+            d["value"] = self.value
+        if self.user_id is not None:
+            d["user_id"] = self.user_id
+        return d
+
+
+@dataclass
+class ReportTraceInput:
+    trace_id: str
+    name: str
+    spans: list[CollectorSpanInput] = field(default_factory=list)
+    status: Literal["pending", "success", "error"] | None = None
+    duration_ms: float | None = None
+    source: Literal["sdk", "api", "cli"] | None = None
+    environment: Literal["production", "staging", "dev"] | None = None
+    metadata: dict[str, Any] | None = None
+    user_feedback: CollectorFeedbackInput | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        d: dict[str, Any] = {
+            "trace_id": self.trace_id,
+            "name": self.name,
+            "spans": [s.to_dict() for s in self.spans],
+        }
+        for key in ("status", "duration_ms", "source", "environment", "metadata"):
+            val = getattr(self, key)
+            if val is not None:
+                d[key] = val
+        if self.user_feedback is not None:
+            d["user_feedback"] = self.user_feedback.to_dict()
+        return d
+
+
+@dataclass
+class ReportTraceOptions:
+    """Options for ``report_trace``."""
+
+    sample_rate: float = 1.0
+
+
+@dataclass
+class ReportTraceResult:
+    sent: bool
+    trace_id: str
+    trace_db_id: int | None = None
+    span_count: int | None = None
+    queued_for_analysis: bool | None = None
+    skip_reason: str | None = None
+
+
+async def report_trace(
+    client: Any,
+    input: ReportTraceInput,
+    options: ReportTraceOptions | None = None,
+) -> ReportTraceResult:
+    """Report a production trace to the collector endpoint.
+
+    Client-side sampling: set ``options.sample_rate`` (0–1).
+    Error traces and thumbs-down feedback bypass sampling.
+    """
+    opts = options or ReportTraceOptions()
+
+    is_error = input.status == "error"
+    is_negative_feedback = input.user_feedback is not None and input.user_feedback.type == "thumbs_down"
+    bypass_sampling = is_error or is_negative_feedback
+
+    if not bypass_sampling and opts.sample_rate < 1.0 and random.random() >= opts.sample_rate:
+        return ReportTraceResult(
+            sent=False,
+            trace_id=input.trace_id,
+            skip_reason="sampled_out",
+        )
+
+    try:
+        response = await client._request("POST", "/api/collector", json=input.to_dict())
+    except Exception as exc:
+        return ReportTraceResult(
+            sent=False,
+            trace_id=input.trace_id,
+            skip_reason=f"request_failed: {exc}",
+        )
+
+    return ReportTraceResult(
+        sent=True,
+        trace_id=input.trace_id,
+        trace_db_id=response.get("trace_db_id"),
+        span_count=response.get("span_count"),
+        queued_for_analysis=response.get("queued_for_analysis"),
+    )

evalgate_sdk/constants.py

@@ -0,0 +1,6 @@
+"""Default constants for the EvalGate SDK.
+
+Port of ``constants.ts``.
+"""
+
+DEFAULT_BASE_URL = "https://api.evalgate.com"

evalgate_sdk/context.py

@@ -0,0 +1,151 @@
+"""Context propagation using Python's contextvars — thread-safe and async-safe."""
+
+from __future__ import annotations
+
+import contextvars
+import functools
+from collections.abc import Callable
+from typing import Any, TypeVar, overload
+
+T = TypeVar("T")
+F = TypeVar("F", bound=Callable[..., Any])
+
+ContextMetadata = dict[str, Any]
+
+_ctx_var: contextvars.ContextVar[ContextMetadata | None] = contextvars.ContextVar("evalai_context", default=None)
+
+
+class EvalContext:
+    """Manages a context scope with metadata propagation."""
+
+    def __init__(self, metadata: ContextMetadata) -> None:
+        self._metadata = dict(metadata)
+        self._token: contextvars.Token[ContextMetadata | None] | None = None
+
+    @property
+    def metadata(self) -> ContextMetadata:
+        return dict(self._metadata)
+
+    def enter(self) -> None:
+        parent = _ctx_var.get()
+        merged = {**(parent or {}), **self._metadata}
+        self._token = _ctx_var.set(merged)
+
+    def exit(self) -> None:
+        if self._token is not None:
+            _ctx_var.reset(self._token)
+            self._token = None
+
+    def __enter__(self) -> EvalContext:
+        self.enter()
+        return self
+
+    def __exit__(self, *args: Any) -> None:
+        self.exit()
+
+    async def __aenter__(self) -> EvalContext:
+        self.enter()
+        return self
+
+    async def __aexit__(self, *args: Any) -> None:
+        self.exit()
+
+
+def create_context(metadata: ContextMetadata) -> EvalContext:
+    """Create a new context scope."""
+    return EvalContext(metadata)
+
+
+def get_current_context() -> ContextMetadata | None:
+    """Get the current context metadata, or None if no context is active."""
+    return _ctx_var.get()
+
+
+def merge_with_context(metadata: dict[str, Any] | None = None) -> dict[str, Any]:
+    """Merge provided metadata with the current context."""
+    current = _ctx_var.get() or {}
+    return {**current, **(metadata or {})}
+
+
+async def with_context(metadata: ContextMetadata, fn: Callable[[], Any]) -> Any:
+    """Run *fn* inside a new context scope (async)."""
+    ctx = create_context(metadata)
+    async with ctx:
+        result = fn()
+        if hasattr(result, "__await__"):
+            return await result
+        return result
+
+
+def with_context_sync(metadata: ContextMetadata, fn: Callable[[], T]) -> T:
+    """Run *fn* inside a new context scope (sync)."""
+    ctx = create_context(metadata)
+    with ctx:
+        return fn()
+
+
+def clone_context(metadata: ContextMetadata) -> ContextMetadata:
+    """Deep-copy metadata."""
+    import copy
+
+    return copy.deepcopy(metadata)
+
+
+def merge_contexts(*contexts: ContextMetadata) -> ContextMetadata:
+    """Merge multiple context dicts left-to-right."""
+    result: ContextMetadata = {}
+    for c in contexts:
+        result.update(c)
+    return result
+
+
+def validate_context(metadata: ContextMetadata) -> None:
+    """Validate context metadata — raises ValueError if invalid."""
+    if not isinstance(metadata, dict):
+        raise ValueError("Context metadata must be a dict")
+    for key in metadata:
+        if not isinstance(key, str):
+            raise ValueError(f"Context keys must be strings, got {type(key)}")
+
+
+class WithContext:
+    """Decorator that wraps a function/method in a context scope.
+
+    Works with both sync and async functions::
+
+        @WithContext({"service": "MyService"})
+        async def process(self, data):
+            ...
+
+        @WithContext({"component": "parser"})
+        def parse(text):
+            ...
+    """
+
+    def __init__(self, metadata: ContextMetadata) -> None:
+        self._metadata = metadata
+
+    @overload
+    def __call__(self, fn: F) -> F: ...
+
+    def __call__(self, fn: Callable[..., Any]) -> Callable[..., Any]:
+        import asyncio
+
+        if asyncio.iscoroutinefunction(fn):
+
+            @functools.wraps(fn)
+            async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
+                ctx = create_context(self._metadata)
+                async with ctx:
+                    return await fn(*args, **kwargs)
+
+            return async_wrapper  # type: ignore[return-value]
+        else:
+
+            @functools.wraps(fn)
+            def sync_wrapper(*args: Any, **kwargs: Any) -> Any:
+                ctx = create_context(self._metadata)
+                with ctx:
+                    return fn(*args, **kwargs)
+
+            return sync_wrapper  # type: ignore[return-value]