langwatch 0.8.1__py3-none-any.whl → 0.9.0__py3-none-any.whl

langwatch/__version__.py

@@ -1,3 +1,3 @@
 """Version information for LangWatch."""
 
-__version__ = "0.8.1" # x-release-please-version
+__version__ = "0.9.0" # x-release-please-version

langwatch/dspy/__init__.py

@@ -737,10 +737,6 @@ class DSPyTracer:
             dspy.Module.__original_call__ = dspy.Module.__call__  # type: ignore
             dspy.Module.__call__ = self.patched_module_call()
 
-        if not hasattr(dspy.Predict, "__original_forward__"):
-            dspy.Predict.__original_forward__ = dspy.Predict.forward  # type: ignore
-            dspy.Predict.forward = self.patched_predict_forward()
-
         language_model_classes = dspy.LM.__subclasses__()
         for lm in language_model_classes:
             if not hasattr(lm, "__original_basic_request__") and hasattr(
@@ -776,7 +772,7 @@ class DSPyTracer:
     def patched_module_call(self):
         self_ = self
 
-        @langwatch.span(ignore_missing_trace_warning=True, type="module")
+        @langwatch.span(ignore_missing_trace_warning=True, type="module", capture_output=False)
         def __call__(self: dspy.Module, *args, **kwargs):
             span = self_.safe_get_current_span()
             signature = (
@@ -801,34 +797,10 @@ class DSPyTracer:
 
         return __call__
 
-    def patched_predict_forward(self):
-        self_ = self
-
-        @langwatch.span(ignore_missing_trace_warning=True, type="module")
-        def forward(self: dspy.Predict, **kwargs):
-            span = self_.safe_get_current_span()
-            signature = kwargs.get("signature", self.signature)
-
-            if span and signature and hasattr(signature, "__name__"):
-                span.update(name=f"{self.__class__.__name__}({signature.__name__})")
-            elif span:
-                span.update(name=f"{self.__class__.__name__}.forward")
-
-            prediction = self.__class__.__original_forward__(self, **kwargs)  # type: ignore
-
-            if span and isinstance(prediction, dspy.Prediction):
-                span.update(output=prediction._store)  # type: ignore
-            elif span:
-                span.update(output=prediction)  # type: ignore
-
-            return prediction
-
-        return forward
-
     def patched_language_model_call(self):
         self_ = self
 
-        @langwatch.span(ignore_missing_trace_warning=True, type="llm")
+        @langwatch.span(ignore_missing_trace_warning=True, type="llm", capture_output=False)
         def call(self: dspy.LM, prompt=None, messages=None, **kwargs):
             all_kwargs = self.kwargs | kwargs
             model = self.model
@@ -895,7 +867,7 @@ class DSPyTracer:
     def patched_legacy_language_model_request(self):
         self_ = self
 
-        @langwatch.span(ignore_missing_trace_warning=True, type="llm")
+        @langwatch.span(ignore_missing_trace_warning=True, type="llm", capture_output=False)
         def basic_request(self: dspy.LM, prompt, **kwargs):
             all_kwargs = self.kwargs | kwargs
             model = all_kwargs.get("model", None)
@@ -947,7 +919,7 @@ class DSPyTracer:
             ) is not getattr(dspy.Retrieve, "forward", None):
                 return self.__class__.__original_forward__(self, *args, **kwargs)  # type: ignore
 
-            @langwatch.span(ignore_missing_trace_warning=True, type="rag")
+            @langwatch.span(ignore_missing_trace_warning=True, type="rag", capture_output=False)
             def forward(self, *args, **kwargs):
                 result = self.__class__.__original_forward__(self, *args, **kwargs)  # type: ignore
 

langwatch/evaluation/__init__.py

@@ -1,9 +1,36 @@
 from typing import Optional
 from langwatch.evaluation.evaluation import Evaluation
-from .evaluation import Evaluation
+from langwatch.evaluation.platform_run import (
+    evaluate,
+    run,  # Deprecated, kept for backwards compatibility
+    EvaluationRunResult,
+    EvaluationRunSummary,
+    EvaluationNotFoundError,
+    EvaluationTimeoutError,
+    EvaluationRunFailedError,
+    EvaluationsApiError,
+    TargetStats,
+    EvaluatorStats,
+)
 
 
 def init(name: str, *, run_id: Optional[str] = None) -> Evaluation:
     evaluation = Evaluation(name, run_id=run_id)
     evaluation.init()
     return evaluation
+
+
+__all__ = [
+    "init",
+    "evaluate",
+    "run",  # Deprecated
+    "Evaluation",
+    "EvaluationRunResult",
+    "EvaluationRunSummary",
+    "EvaluationNotFoundError",
+    "EvaluationTimeoutError",
+    "EvaluationRunFailedError",
+    "EvaluationsApiError",
+    "TargetStats",
+    "EvaluatorStats",
+]

langwatch/evaluation/evaluation.py

@@ -1,13 +1,15 @@
 from __future__ import annotations
 import asyncio
 from contextlib import contextmanager
+from contextvars import ContextVar
+from dataclasses import dataclass
 import json
 import threading
 import time
 import traceback
 import httpx
 import pandas as pd
-from opentelemetry import trace
+from opentelemetry import trace, context as otel_context
 from opentelemetry.trace import Span
 from pydantic import BaseModel, Field
 from typing import (
@@ -43,6 +45,35 @@ from concurrent.futures import Future, ThreadPoolExecutor, as_completed
 
 _tracer = trace.get_tracer(__name__)
 
+
+@dataclass
+class TargetContext:
+    """Context for the current target() execution."""
+
+    target_id: str
+    index: int
+    trace_id: str
+    predicted: Optional[Dict[str, Any]] = None  # Set via log_response()
+
+
+@dataclass
+class IterationContext:
+    """Context for the current iteration (index + item)."""
+
+    index: int
+    item: Any
+
+
+# ContextVar for target context isolation (works across threads)
+_target_context: ContextVar[Optional[TargetContext]] = ContextVar(
+    "_target_context", default=None
+)
+
+# ContextVar for iteration context (index + item) - thread-safe
+_iteration_context: ContextVar[Optional[IterationContext]] = ContextVar(
+    "_iteration_context", default=None
+)
+
 ItemT = TypeVar("ItemT")
 
 
@@ -65,11 +96,24 @@ class EvaluationResult(BaseModel):
     traceback: Optional[List[str]] = Field(
         description="Traceback information for debugging", default=None
     )
+    target_id: Optional[str] = Field(
+        default=None, description="ID of the target this evaluation is for"
+    )
+
+
+class TargetInfo(BaseModel):
+    """Represents a registered target with its metadata."""
+
+    id: str
+    name: str
+    type: Literal["prompt", "agent", "custom"] = "custom"
+    metadata: Optional[Dict[str, Union[str, int, float, bool]]] = None
 
 
 class Batch(TypedDict):
     dataset: List[BatchEntry]
     evaluations: List[EvaluationResult]
+    targets: List[TargetInfo]
 
 
 class BatchEntry(BaseModel):
@@ -78,6 +122,9 @@ class BatchEntry(BaseModel):
     duration: int
     error: Optional[str] = None
     trace_id: str
+    target_id: Optional[str] = None
+    cost: Optional[float] = None
+    predicted: Optional[Dict[str, Any]] = None
 
 
 class IterationInfo(TypedDict):
@@ -105,12 +152,26 @@ class Evaluation:
 
         # Sending results
         self.lock = threading.Lock()
-        self.batch: Batch = {"dataset": [], "evaluations": []}
+        self.batch: Batch = {"dataset": [], "evaluations": [], "targets": []}
         self.last_sent = 0
         self.debounce_interval = 1  # 1 second
         self.threads: List[threading.Thread] = []
         self.initialized = False
 
+        # Target registry - tracks registered targets and their metadata
+        self._targets: Dict[str, TargetInfo] = {}
+
+        # Track whether with_target() was used in the current iteration
+        # If so, we don't create row-level dataset entries
+        self._current_iteration_used_with_target = False
+
+        # Track whether target() has EVER been used in this evaluation
+        # Once set to True, we stop creating iteration-level traces
+        self._evaluation_uses_targets: bool = False
+
+        # Store the active iteration trace so target() can close it early
+        self._active_iteration_trace: Optional[LangWatchTrace] = None
+
     def init(self):
         if not langwatch.get_api_key():
             raise ValueError(
@@ -233,11 +294,25 @@ class Evaluation:
         item: Any,
         in_thread: bool = False,
     ) -> Iterator[Any]:
-        # Iteration will be None if we find ourselves in a parallel loop, but still
-        # in the phase of collecting the evaluation.submit() processes. When in_thread,
-        # then it's when we actually collect the iteration info.
-        iteration = (
-            IterationInfo(
+        # Reset with_target tracking for this iteration
+        self._current_iteration_used_with_target = False
+
+        # Set iteration context (thread-safe via contextvars)
+        # This allows target() to access index/item without race conditions
+        iter_ctx = IterationContext(index=index, item=item)
+        iter_token = _iteration_context.set(iter_ctx)
+
+        # Determine if we should create an iteration trace:
+        # - Don't create if evaluation uses targets (each target creates its own trace)
+        # - Don't create if we're collecting submit() calls (not in_thread yet)
+        should_create_iteration_trace = (
+            not self._evaluation_uses_targets
+            and (in_thread or len(self._futures) == 0)
+        )
+
+        iteration: Optional[IterationInfo] = None
+        if should_create_iteration_trace:
+            iteration = IterationInfo(
                 trace=langwatch.trace(
                     name="evaluation.loop_iteration",
                     metadata={
@@ -250,12 +325,9 @@ class Evaluation:
                 duration=0,
                 error=None,
             )
-            if in_thread or len(self._futures) == 0
-            else None
-        )
-
-        if iteration is not None:
             iteration["trace"].__enter__()
+            # Store for target() to potentially close early
+            self._active_iteration_trace = iteration["trace"]
 
         start_time = time.time()
         try:
@@ -265,8 +337,13 @@ class Evaluation:
                 iteration["error"] = e
             print(f"\n[Evaluation Error] index={index}")
             traceback.print_exc()
+        finally:
+            # Reset iteration context
+            _iteration_context.reset(iter_token)
 
-        if iteration is not None:
+        # Handle iteration trace cleanup
+        # Note: If target() was used, it may have already closed the trace
+        if iteration is not None and not self._evaluation_uses_targets:
             try:
                 iteration["duration"] = int((time.time() - start_time) * 1000)
 
@@ -274,7 +351,9 @@ class Evaluation:
                 # from being added to the batch and change the trace name
                 if not in_thread and len(self._futures) > 0:
                     iteration["trace"].update(name="evaluation.loop")
-                else:
+                # Only add row-level entry if with_target was NOT used
+                # When with_target is used, it creates per-target dataset entries instead
+                elif not self._current_iteration_used_with_target:
                     self._add_to_batch(iteration)
 
                 if iteration["error"] is not None:
@@ -284,6 +363,9 @@ class Evaluation:
             finally:
                 iteration["trace"].__exit__(None, None, None)
 
+        # Clear active iteration trace reference
+        self._active_iteration_trace = None
+
     def _add_to_batch(self, iteration: IterationInfo):
         entry: Any = (
             iteration["item"].to_dict()
@@ -327,6 +409,7 @@ class Evaluation:
             if (
                 len(self.batch["dataset"]) == 0
                 and len(self.batch["evaluations"]) == 0
+                and len(self.batch["targets"]) == 0
                 and not finished
             ):
                 return
@@ -340,7 +423,13 @@ class Evaluation:
                     del eval_["data"]
                 evaluations.append(eval_)
 
-            body = {
+            # Build targets array for API
+            targets = [
+                target.model_dump(exclude_none=True, exclude_unset=True)
+                for target in self.batch["targets"]
+            ]
+
+            body: Dict[str, Any] = {
                 "experiment_slug": self.experiment_slug,
                 "name": f"{self.name}",
                 "run_id": self.run_id,
@@ -356,6 +445,10 @@ class Evaluation:
                 },
             }
 
+            # Only include targets if we have any
+            if len(targets) > 0:
+                body["targets"] = targets
+
             if finished:
                 if not isinstance(body["timestamps"], dict):
                     body["timestamps"] = {}
@@ -370,7 +463,7 @@ class Evaluation:
             self.threads.append(thread)
 
             # Clear the batch and update the last sent time
-            self.batch = {"dataset": [], "evaluations": []}
+            self.batch = {"dataset": [], "evaluations": [], "targets": []}
             self.last_sent = time.time()
 
     @classmethod
@@ -402,6 +495,261 @@ class Evaluation:
 
         asyncio.run(wait_for_completion(self))
 
+    def _register_target(
+        self,
+        target: str,
+        metadata: Optional[Dict[str, Union[str, int, float, bool]]] = None,
+    ) -> str:
+        """
+        Register a target with its metadata. Returns the target ID.
+
+        If the target was already registered:
+        - If no new metadata is provided, the existing target is used
+        - If new metadata is provided and differs from existing, raises an error
+
+        Args:
+            target: The target name/ID
+            metadata: Optional metadata for this target (model, temperature, etc.)
+
+        Returns:
+            The target ID
+        """
+        with self.lock:
+            if target in self._targets:
+                existing = self._targets[target]
+                if metadata is not None:
+                    # Check if metadata matches
+                    existing_meta = existing.metadata or {}
+                    if existing_meta != metadata:
+                        raise ValueError(
+                            f"Target '{target}' was previously registered with different metadata.\n"
+                            f"Original: {existing_meta}\n"
+                            f"New: {metadata}\n"
+                            f"If you want to use different metadata, please use a different target name."
+                        )
+                return target
+
+            # Register new target
+            target_info = TargetInfo(
+                id=target,
+                name=target,
+                type="custom",
+                metadata=metadata,
+            )
+            self._targets[target] = target_info
+            self.batch["targets"].append(target_info)
+            return target
+
+    @contextmanager
+    def target(
+        self,
+        name: str,
+        metadata: Optional[Dict[str, Union[str, int, float, bool]]] = None,
+    ) -> Iterator[None]:
+        """
+        Context manager for executing code within a target context.
+
+        Creates a dataset entry for this specific target execution, capturing
+        duration automatically. This enables proper per-target latency tracking
+        when comparing multiple models/configurations.
+
+        Each target() call creates its own independent trace, allowing you to
+        view execution details separately for each model/configuration.
+
+        Inside this context, log() calls will automatically use this target
+        unless an explicit target is provided.
+
+        Args:
+            name: Unique identifier for the target
+            metadata: Optional metadata for comparison (e.g., {"model": "gpt-4"})
+
+        Example:
+            ```python
+            for index, row in evaluation.loop(df.iterrows()):
+                def task(index, row):
+                    # Compare GPT-4 and Claude
+                    with evaluation.target("gpt-4", {"model": "openai/gpt-4"}):
+                        response = call_gpt4(row["question"])
+                        # target auto-inferred, use data= to record output
+                        evaluation.log("quality", index=index, score=0.95,
+                                       data={"output": response})
+
+                    with evaluation.target("claude", {"model": "anthropic/claude"}):
+                        response = call_claude(row["question"])
+                        evaluation.log("quality", index=index, score=0.85,
+                                       data={"output": response})
+
+                evaluation.submit(task, index, row)
+            ```
+        """
+        # On FIRST target() call ever in this evaluation:
+        # - Set flag to skip creating iteration-level traces going forward
+        # - Close the active iteration trace if any (it won't have useful content)
+        if not self._evaluation_uses_targets:
+            self._evaluation_uses_targets = True
+            # Close the active iteration trace early
+            if self._active_iteration_trace is not None:
+                self._active_iteration_trace.__exit__(None, None, None)
+                self._active_iteration_trace = None
+
+        # Mark that target() was used in this iteration (for dataset entry logic)
+        self._current_iteration_used_with_target = True
+
+        # Register target
+        self._register_target(name, metadata)
+
+        # Get index and item from iteration context (thread-safe via contextvars)
+        # This prevents race conditions when multiple threads are running evaluations
+        iter_ctx = _iteration_context.get()
+        if iter_ctx is not None:
+            index = iter_ctx.index
+            current_item = iter_ctx.item
+        else:
+            # Fallback to instance variables (for backwards compatibility / direct usage)
+            index = self._current_index
+            current_item = self._current_item
+
+        target_trace: Optional[LangWatchTrace] = None
+        start_time = time.time()
+        error_occurred: Optional[Exception] = None
+        trace_id = ""
+
+        # Set up context for log() inference
+        ctx = TargetContext(
+            target_id=name,
+            index=index,
+            trace_id="",  # Will be set after entering trace
+        )
+        target_context_token = _target_context.set(ctx)
+
+        try:
+            # Create an INDEPENDENT root trace for this target
+            # We use a new tracer without any parent context to get a unique trace_id
+            # The key is using the tracer directly with context=None to prevent
+            # parent context inheritance
+            from opentelemetry.sdk.trace import TracerProvider
+            from opentelemetry.trace import INVALID_SPAN_CONTEXT
+
+            tracer = trace.get_tracer("langwatch-evaluation")
+
+            # Start a new root span with no parent by passing an empty context
+            # This ensures each target gets a unique trace_id
+            root_context = otel_context.Context()
+
+            with tracer.start_as_current_span(
+                f"evaluation.target.{name}",
+                context=root_context,
+                attributes={
+                    "evaluation.run_id": self.run_id,
+                    "evaluation.index": index,
+                    "evaluation.target": name,
+                },
+            ) as span:
+                span_context = span.get_span_context()
+                trace_id = format(span_context.trace_id, "032x")
+                ctx.trace_id = trace_id
+
+                try:
+                    yield
+                except Exception as e:
+                    error_occurred = e
+                    raise
+
+        except Exception as e:
+            if error_occurred is None:
+                error_occurred = e
+            raise
+        finally:
+            duration_ms = int((time.time() - start_time) * 1000)
+
+            # Create dataset entry for this target
+            # Use the captured current_item, NOT self._current_item (which may have changed)
+            entry_data: Any = (
+                current_item.to_dict()
+                if hasattr(current_item, "to_dict")
+                else (
+                    current_item.__dict__
+                    if hasattr(current_item, "__dict__")
+                    else (
+                        current_item[1].to_dict()
+                        if type(current_item) == tuple
+                        and hasattr(current_item[1], "to_dict")
+                        else (
+                            current_item[1].__dict__
+                            if type(current_item) == tuple
+                            and hasattr(current_item[1], "__dict__")
+                            else {
+                                "entry": json.dumps(
+                                    current_item, cls=SerializableWithStringFallback
+                                )
+                            }
+                        )
+                    )
+                )
+            )
+
+            # Get predicted output from context (set via log_response())
+            predicted = ctx.predicted
+
+            batch_entry = BatchEntry(
+                index=index,
+                entry=entry_data,
+                duration=duration_ms,
+                error=str(error_occurred) if error_occurred else None,
+                trace_id=trace_id,
+                target_id=name,
+                predicted=predicted,
+            )
+
+            with self.lock:
+                self.batch["dataset"].append(batch_entry)
+
+            # Reset target context
+            _target_context.reset(target_context_token)
+
+            # Schedule send
+            if time.time() - self.last_sent >= self.debounce_interval:
+                self._send_batch()
+
+    def log_response(self, response: Union[str, Dict[str, Any]]) -> None:
+        """
+        Log the model's response/output for the current target.
+
+        Must be called inside a `target()` context. The response will be stored
+        in the dataset entry's `predicted` field, which is displayed in the
+        results table.
+
+        Args:
+            response: The model's output. Can be a string (will be wrapped as
+                     {"output": response}) or a dict with named outputs.
+
+        Example:
+            ```python
+            with evaluation.target("gpt-4", {"model": "openai/gpt-4"}):
+                response = call_gpt4(row["question"])
+                evaluation.log_response(response)  # Store the output
+                evaluation.log("quality", index=index, score=0.95)  # Log metrics
+            ```
+
+        Raises:
+            RuntimeError: If called outside of a target() context.
+        """
+        ctx = _target_context.get()
+        if ctx is None:
+            raise RuntimeError(
+                "log_response() must be called inside a target() context. "
+                "Example: with evaluation.target('my-target'): evaluation.log_response(response)"
+            )
+
+        # Normalize response to dict format
+        if isinstance(response, str):
+            ctx.predicted = {"output": response}
+        elif isinstance(response, dict):
+            ctx.predicted = response
+        else:
+            # Try to convert to string for other types
+            ctx.predicted = {"output": str(response)}
+
     def log(
         self,
         metric: str,
@@ -415,17 +763,57 @@ class Evaluation:
         duration: Optional[int] = None,
         cost: Optional[Money] = None,
         error: Optional[Exception] = None,
+        target: Optional[str] = None,
+        metadata: Optional[Dict[str, Union[str, int, float, bool]]] = None,
     ):
+        """
+        Log an evaluation metric result.
+
+        Args:
+            metric: Name of the metric being logged
+            index: Row index in the dataset (must be an integer)
+            data: Additional data/inputs for the evaluation
+            score: Numeric score (0-1 typically)
+            passed: Whether the evaluation passed
+            label: Label/category for the result
+            details: Human-readable description of the result
+            status: Status of the evaluation ("processed", "error", "skipped")
+            duration: Duration in milliseconds
+            cost: Cost of the evaluation
+            error: Exception if an error occurred
+            target: Optional target name for multi-target comparisons.
+                    First call with a target name registers it with the provided metadata.
+                    Subsequent calls with the same target can omit metadata.
+                    If called inside with_target(), the target is auto-inferred from context.
+            metadata: Optional metadata for the target (model, temperature, etc.).
+                      Only used on the first call for each target.
+                      Raises error if conflicting metadata is provided for same target.
+        """
         try:
             index_ = int(cast(Any, index))
         except Exception:
             raise ValueError(f"Index must be an integer, got {index}")
 
+        # Get target context (if inside with_target)
+        ctx = _target_context.get()
+
+        # Use context target if not explicitly provided
+        effective_target = target if target is not None else (ctx.target_id if ctx else None)
+
+        # Register target if provided (explicit or from context)
+        target_id: Optional[str] = None
+        if effective_target is not None:
+            target_id = self._register_target(effective_target, metadata)
+
+        # Use trace_id from context if available
+        trace_id = (
+            ctx.trace_id
+            if ctx
+            else format(trace.get_current_span().get_span_context().trace_id, "x")
+        )
+
         eval = EvaluationResult(
-            trace_id=format(
-                trace.get_current_span().get_span_context().trace_id,
-                "x",
-            ),
+            trace_id=trace_id,
             name=metric,
             evaluator=metric,
             status=status if status else "error" if error else "processed",
@@ -443,6 +831,7 @@ class Evaluation:
                 if error
                 else None
             ),
+            target_id=target_id,
         )
 
         with self.lock:

langwatch/evaluation/platform_run.py

@@ -0,0 +1,462 @@
+"""
+Runner for platform-configured evaluations (Evaluations V3).
+
+This module provides the `run()` function to execute evaluations that are
+configured in the LangWatch platform from CI/CD pipelines or scripts.
+"""
+
+from dataclasses import dataclass, field
+from typing import Callable, List, Literal, Optional
+from urllib.parse import urlparse, urlunparse
+import sys
+import time
+import httpx
+
+import langwatch
+from langwatch.state import get_api_key, get_endpoint
+
+
+def _replace_url_domain(url: str, new_base: str) -> str:
+    """Replace the domain/scheme of a URL with a new base URL, preserving the path."""
+    if not url:
+        return url
+
+    parsed_url = urlparse(url)
+    parsed_new_base = urlparse(new_base)
+
+    # Replace scheme and netloc with new base, keep path/query/fragment
+    return urlunparse((
+        parsed_new_base.scheme,
+        parsed_new_base.netloc,
+        parsed_url.path,
+        parsed_url.params,
+        parsed_url.query,
+        parsed_url.fragment,
+    ))
+
+
+class EvaluationNotFoundError(Exception):
+    """Raised when evaluation slug doesn't exist."""
+
+    def __init__(self, slug: str):
+        self.slug = slug
+        super().__init__(f"Evaluation not found: {slug}")
+
+
+class EvaluationTimeoutError(Exception):
+    """Raised when evaluation run times out."""
+
+    def __init__(self, run_id: str, progress: int, total: int):
+        self.run_id = run_id
+        self.progress = progress
+        self.total = total
+        super().__init__(
+            f"Evaluation run timed out: {run_id} ({progress}/{total} completed)"
+        )
+
+
+class EvaluationRunFailedError(Exception):
+    """Raised when evaluation run fails."""
+
+    def __init__(self, run_id: str, error: str):
+        self.run_id = run_id
+        self.error_message = error
+        super().__init__(f"Evaluation run failed: {error}")
+
+
+class EvaluationsApiError(Exception):
+    """Raised for other API errors."""
+
+    def __init__(self, message: str, status_code: int):
+        self.status_code = status_code
+        super().__init__(message)
+
+
+@dataclass
+class TargetStats:
+    """Statistics for a single target."""
+
+    target_id: str
+    name: str
+    passed: int
+    failed: int
+    avg_latency: float
+    total_cost: float
+
+
+@dataclass
+class EvaluatorStats:
+    """Statistics for a single evaluator."""
+
+    evaluator_id: str
+    name: str
+    passed: int
+    failed: int
+    pass_rate: float
+    avg_score: Optional[float] = None
+
+
+@dataclass
+class EvaluationRunSummary:
+    """Summary of a completed evaluation run."""
+
+    run_id: str
+    total_cells: int
+    completed_cells: int
+    failed_cells: int
+    duration: int
+    run_url: str = ""
+    targets: List[TargetStats] = field(default_factory=list)
+    evaluators: List[EvaluatorStats] = field(default_factory=list)
+    total_passed: int = 0
+    total_failed: int = 0
+    pass_rate: float = 0.0
+    total_cost: float = 0.0
+
+
+@dataclass
+class EvaluationRunResult:
+    """Result of running a platform evaluation."""
+
+    run_id: str
+    status: Literal["completed", "failed", "stopped"]
+    passed: int
+    failed: int
+    pass_rate: float
+    duration: int
+    run_url: str
+    summary: EvaluationRunSummary
+
+    def print_summary(self, exit_on_failure: Optional[bool] = None) -> None:
+        """
+        Print a CI-friendly summary and optionally exit with code 1 on failure.
+
+        Args:
+            exit_on_failure: If True, calls sys.exit(1) when there are failures.
+                           If False, never exits.
+                           If None (default), auto-detects: exits in scripts/CI, doesn't exit in notebooks.
+        """
+        _print_summary(self)
+
+        # Auto-detect: don't exit in notebooks, exit in scripts/CI
+        should_exit = exit_on_failure if exit_on_failure is not None else not _is_notebook()
+
+        if should_exit and self.failed > 0:
+            sys.exit(1)
+
+
+def _is_notebook() -> bool:
+    """Detect if running in a Jupyter notebook."""
+    try:
+        from IPython import get_ipython  # type: ignore
+
+        shell = get_ipython().__class__.__name__
+        if shell == "ZMQInteractiveShell":
+            return True  # Jupyter notebook or qtconsole
+        elif shell == "TerminalInteractiveShell":
+            return False  # Terminal running IPython
+        else:
+            return False
+    except (ImportError, AttributeError, NameError):
+        return False
+
+
+def evaluate(
+    slug: str,
+    *,
+    poll_interval: float = 2.0,
+    timeout: float = 600.0,
+    on_progress: Optional[Callable[[int, int], None]] = None,
+    api_key: Optional[str] = None,
+) -> EvaluationRunResult:
+    """
+    Run a platform-configured evaluation and wait for completion.
+
+    This runs an Evaluation that you have configured in the LangWatch platform.
+    The evaluation will execute all targets and evaluators defined in the configuration.
+
+    Args:
+        slug: The slug of the evaluation to run (found in the evaluation URL)
+        poll_interval: Seconds between status checks (default: 2.0)
+        timeout: Maximum seconds to wait for completion (default: 600.0 = 10 minutes)
+        on_progress: Optional callback for progress updates (completed, total)
+        api_key: Optional API key override (uses LANGWATCH_API_KEY env var by default)
+
+    Returns:
+        EvaluationRunResult with pass rate and summary. Call result.print_summary()
+        to display results and exit with code 1 on failure.
+
+    Raises:
+        EvaluationNotFoundError: If the evaluation slug doesn't exist
+        EvaluationTimeoutError: If the evaluation doesn't complete within timeout
+        EvaluationRunFailedError: If the evaluation fails
+        EvaluationsApiError: For other API errors
+
+    Example:
+        ```python
+        import langwatch
+
+        result = langwatch.evaluation.evaluate("my-evaluation-slug")
+        result.print_summary()
+        ```
+    """
+    langwatch.ensure_setup()
+
+    effective_api_key = api_key or get_api_key()
+    endpoint = get_endpoint()
+
+    if not effective_api_key:
+        raise ValueError(
+            "API key not set. Set LANGWATCH_API_KEY environment variable or pass api_key parameter."
+        )
+
+    # Start the run
+    start_response = _start_run(slug, endpoint, effective_api_key)
+    run_id = start_response["runId"]
+    total = start_response.get("total", 0)
+
+    # Use the run URL from API but replace domain with configured endpoint
+    api_run_url = start_response.get("runUrl", "")
+    run_url = _replace_url_domain(api_run_url, endpoint) if api_run_url else ""
+
+    print(f"Started evaluation run: {run_id}")
+    if run_url:
+        print(f"Follow live: {run_url}")
+
+    # Track last progress for change detection
+    last_progress = 0
+
+    # Print initial progress
+    if total > 0:
+        print(f"Progress: 0/{total} (0%)", end="", flush=True)
+    if on_progress:
+        on_progress(0, total)
+
+    # Poll until complete
+    start_time = time.time()
+    while True:
+        if time.time() - start_time > timeout:
+            print()  # Newline after progress
+            status = _get_run_status(run_id, endpoint, effective_api_key)
+            raise EvaluationTimeoutError(
+                run_id, status.get("progress", 0), status.get("total", 0)
+            )
+
+        time.sleep(poll_interval)
+
+        status = _get_run_status(run_id, endpoint, effective_api_key)
+        progress = status.get("progress", 0)
+        total = status.get("total", total)
+
+        # Update progress display if changed
+        if progress != last_progress and total > 0:
+            percentage = (progress / total) * 100
+            # Use carriage return to overwrite the line
+            print(f"\rProgress: {progress}/{total} ({percentage:.0f}%)", end="", flush=True)
+            last_progress = progress
+
+        if on_progress:
+            on_progress(progress, total)
+
+        run_status = status.get("status")
+
+        if run_status == "completed":
+            print()  # Newline after progress
+            summary_data = status.get("summary", {})
+            return _build_result(run_id, "completed", summary_data, run_url)
+
+        if run_status == "failed":
+            print()  # Newline after progress
+            raise EvaluationRunFailedError(
+                run_id, status.get("error", "Unknown error")
+            )
+
+        if run_status == "stopped":
+            print()  # Newline after progress
+            summary_data = status.get("summary", {})
+            return _build_result(run_id, "stopped", summary_data, run_url)
+
+
+def _start_run(slug: str, endpoint: str, api_key: str) -> dict:
+    """Start an evaluation run."""
+    with httpx.Client(timeout=60) as client:
+        response = client.post(
+            f"{endpoint}/api/evaluations/v3/{slug}/run",
+            headers={"X-Auth-Token": api_key},
+        )
+
+    if response.status_code == 404:
+        raise EvaluationNotFoundError(slug)
+    if response.status_code == 401:
+        raise EvaluationsApiError("Unauthorized - check your API key", 401)
+    if not response.is_success:
+        error_body = response.json() if response.content else {}
+        raise EvaluationsApiError(
+            error_body.get("error", f"Failed to start evaluation: {response.status_code}"),
+            response.status_code,
+        )
+
+    return response.json()
+
+
+def _get_run_status(run_id: str, endpoint: str, api_key: str) -> dict:
+    """Get the status of a run."""
+    with httpx.Client(timeout=60) as client:
+        response = client.get(
+            f"{endpoint}/api/evaluations/v3/runs/{run_id}",
+            headers={"X-Auth-Token": api_key},
+        )
+
+    if response.status_code == 404:
+        raise EvaluationsApiError(f"Run not found: {run_id}", 404)
+    if response.status_code == 401:
+        raise EvaluationsApiError("Unauthorized - check your API key", 401)
+    if not response.is_success:
+        error_body = response.json() if response.content else {}
+        raise EvaluationsApiError(
+            error_body.get("error", f"Failed to get run status: {response.status_code}"),
+            response.status_code,
+        )
+
+    return response.json()
+
+
+def _build_result(
+    run_id: str,
+    status: Literal["completed", "failed", "stopped"],
+    summary_data: dict,
+    run_url: str,
+) -> EvaluationRunResult:
+    """Build the result object from API response."""
+    total_cells = summary_data.get("totalCells", 0)
+    completed_cells = summary_data.get("completedCells", 0)
+    failed_cells = summary_data.get("failedCells", 0)
+    duration = summary_data.get("duration", 0)
+
+    total_passed = summary_data.get("totalPassed", completed_cells - failed_cells)
+    total_failed = summary_data.get("totalFailed", failed_cells)
+    pass_rate = summary_data.get(
+        "passRate",
+        (total_passed / completed_cells * 100) if completed_cells > 0 else 0.0,
+    )
+
+    # Parse targets
+    targets: List[TargetStats] = []
+    for t in summary_data.get("targets", []):
+        targets.append(
+            TargetStats(
+                target_id=t.get("targetId", ""),
+                name=t.get("name", ""),
+                passed=t.get("passed", 0),
+                failed=t.get("failed", 0),
+                avg_latency=t.get("avgLatency", 0),
+                total_cost=t.get("totalCost", 0),
+            )
+        )
+
+    # Parse evaluators
+    evaluators: List[EvaluatorStats] = []
+    for e in summary_data.get("evaluators", []):
+        evaluators.append(
+            EvaluatorStats(
+                evaluator_id=e.get("evaluatorId", ""),
+                name=e.get("name", ""),
+                passed=e.get("passed", 0),
+                failed=e.get("failed", 0),
+                pass_rate=e.get("passRate", 0),
+                avg_score=e.get("avgScore"),
+            )
+        )
+
+    summary = EvaluationRunSummary(
+        run_id=run_id,
+        total_cells=total_cells,
+        completed_cells=completed_cells,
+        failed_cells=failed_cells,
+        duration=duration,
+        run_url=run_url,  # Always use the endpoint-based URL we constructed
+        targets=targets,
+        evaluators=evaluators,
+        total_passed=total_passed,
+        total_failed=total_failed,
+        pass_rate=pass_rate,
+        total_cost=summary_data.get("totalCost", 0),
+    )
+
+    return EvaluationRunResult(
+        run_id=run_id,
+        status=status,
+        passed=total_passed,
+        failed=total_failed,
+        pass_rate=pass_rate,
+        duration=duration,
+        run_url=summary.run_url,
+        summary=summary,
+    )
+
+
+def _print_summary(result: EvaluationRunResult) -> None:
+    """Print a CI-friendly summary of the evaluation results."""
+    summary = result.summary
+
+    print("\n" + "═" * 60)
+    print("  EVALUATION RESULTS")
+    print("═" * 60)
+    print(f"  Run ID:     {result.run_id}")
+    print(f"  Status:     {result.status.upper()}")
+    print(f"  Duration:   {result.duration / 1000:.1f}s")
+    print("─" * 60)
+    print(f"  Passed:     {result.passed}")
+    print(f"  Failed:     {result.failed}")
+    print(f"  Pass Rate:  {result.pass_rate:.1f}%")
+
+    if summary.targets:
+        print("─" * 60)
+        print("  TARGETS:")
+        for target in summary.targets:
+            print(f"    {target.name}: {target.passed} passed, {target.failed} failed")
+            if target.avg_latency:
+                print(f"      Avg latency: {target.avg_latency:.0f}ms")
+            if target.total_cost:
+                print(f"      Total cost: ${target.total_cost:.4f}")
+
+    if summary.evaluators:
+        print("─" * 60)
+        print("  EVALUATORS:")
+        for evaluator in summary.evaluators:
+            print(f"    {evaluator.name}: {evaluator.pass_rate:.1f}% pass rate")
+            if evaluator.avg_score is not None:
+                print(f"      Avg score: {evaluator.avg_score:.2f}")
+
+    print("─" * 60)
+    print(f"  View details: {result.run_url}")
+    print("═" * 60 + "\n")
+
+
+def run(
+    slug: str,
+    *,
+    poll_interval: float = 2.0,
+    timeout: float = 600.0,
+    on_progress: Optional[Callable[[int, int], None]] = None,
+    api_key: Optional[str] = None,
+) -> EvaluationRunResult:
+    """
+    Deprecated: Use `evaluate()` instead.
+
+    Run a platform-configured evaluation and wait for completion.
+    """
+    import warnings
+
+    warnings.warn(
+        "langwatch.evaluation.run() is deprecated, use langwatch.evaluation.evaluate() instead",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    return evaluate(
+        slug,
+        poll_interval=poll_interval,
+        timeout=timeout,
+        on_progress=on_progress,
+        api_key=api_key,
+    )

{langwatch-0.8.1.dist-info → langwatch-0.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langwatch
-Version: 0.8.1
+Version: 0.9.0
 Summary: LangWatch Python SDK, for monitoring your LLMs
 Author-email: Langwatch Engineers <engineering@langwatch.ai>
 License: MIT

{langwatch-0.8.1.dist-info → langwatch-0.9.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 langwatch/__init__.py,sha256=GMq4SV2Tz2i0JD05shqnw2lBW5cgMx4Zzo141hp106k,4266
-langwatch/__version__.py,sha256=l2r_v6gqH58S38dAeIr-BCiWrh25Ql4biGJMjTpZZ1o,91
+langwatch/__version__.py,sha256=sympc_lD0EH1ffjgsP80P8i4Sqm2XBcIgblEeQTq6bs,91
 langwatch/attributes.py,sha256=nXdI_G85wQQCAdAcwjCiLYdEYj3wATmfgCmhlf6dVIk,3910
 langwatch/batch_evaluation.py,sha256=Y_S3teXpHV07U-vvJYyV1PB6d0CgyFM_rTzPp6GnEBo,16165
 langwatch/client.py,sha256=WTNcYSik7kZ2kH-qGDnhbMTosc8e_Xhab_lZlfh5TC8,25559
@@ -15,9 +15,10 @@ langwatch/tracer.py,sha256=t5FOdP1es9H_pPGqGUBLXCyEln0tTi4m4M9b6WxCrPU,975
 langwatch/types.py,sha256=h6r3tNTzWqENx-9j_JPmOMZfFoKq9SNpEtxpAACk2G0,3114
 langwatch/dataset/__init__.py,sha256=hZBcbjXuBO2qE5osJtd9wIE9f45F6-jpNTrne5nk4eE,2606
 langwatch/domain/__init__.py,sha256=gSCOV3WkRhp_--9D1vxw7BYpnMRbpGh-2NbsXd4KZC0,6074
-langwatch/dspy/__init__.py,sha256=F35iLwiznMJPgXLVYOvybjDWxdYlSN4vn3EzxC27Awc,34054
-langwatch/evaluation/__init__.py,sha256=Jy7PW5VQbMoDGdOLRlQmDEvo_9TDkBLmrLrfocxddLM,281
-langwatch/evaluation/evaluation.py,sha256=hmtY7rfgJm4TbTEMUP_x89B2L_Jyi7aNGhjNUxw1N4A,16112
+langwatch/dspy/__init__.py,sha256=wp8AmobV8XGVWOI8MQFmXPHu-8Wq3wvjB6YiHQm9Fdg,33007
+langwatch/evaluation/__init__.py,sha256=dctG-Ec0N_Or2Ta0XW6liYtdpMZN3ZtRXqUoeG5ksnk,870
+langwatch/evaluation/evaluation.py,sha256=MqMiGlsPIS5zqN1wKfhEs6mIGLRwB452iqDTSQFbtYo,31735
+langwatch/evaluation/platform_run.py,sha256=cwuRNtG99nhvqGL-YoOwdvEH3x-hDaVUzl7Vx9orjPo,14546
 langwatch/exporters/filterable_batch_span_exporter.py,sha256=MlhZjui56XD6p2sa8kEGyr-Hb3wqudknngmemnB4Twg,2142
 langwatch/generated/langwatch_rest_api_client/__init__.py,sha256=8r-9pAj7fK7vnVX3mT0y_zS4B9ZRqD6RZiBo5fPra60,156
 langwatch/generated/langwatch_rest_api_client/client.py,sha256=o_mdLqyBCQstu5tS1WZFwqIEbGwkvWQ7eQjuCJw_5VY,12419
@@ -415,6 +416,6 @@ langwatch/utils/initialization.py,sha256=1KoZmkHOvGEVF0j-4t4xRQdA_2C_SPiF7qFXqEG
 langwatch/utils/module.py,sha256=KLBNOK3mA9gCSifCcQX_lOtU48BJQDWvFKtF6NMvwVA,688
 langwatch/utils/transformation.py,sha256=76MGXyrYTxM0Yri36NJqLK-XxL4BBYdmKWAXXlw3D4Q,7690
 langwatch/utils/utils.py,sha256=ZCOSie4o9LdJ7odshNfCNjmgwgQ27ojc5ENqt1rXuSs,596
-langwatch-0.8.1.dist-info/METADATA,sha256=osaR4n3f3-Uo3PhYP_Dox70Dgs5fiCBnOEpu4LAhTVQ,13192
-langwatch-0.8.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-langwatch-0.8.1.dist-info/RECORD,,
+langwatch-0.9.0.dist-info/METADATA,sha256=JtLLtVbyy0iau3ySelLpMO4RpjrQAEyhd72J9NkxHl8,13192
+langwatch-0.9.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+langwatch-0.9.0.dist-info/RECORD,,