opik 1.9.26__py3-none-any.whl → 1.9.41__py3-none-any.whl

opik/evaluation/models/litellm/util.py

@@ -36,16 +36,33 @@ def apply_model_specific_filters(
     already_warned: Set[str],
     warn: Callable[[str, Any], None],
 ) -> None:
-    """Remove parameters known to be unsupported for specific models.
+    """Adjust/drop params for specific model families before calling LiteLLM.
 
-    Currently handles the GPT-5 family which only honours temperature=1 and does not
-    return log probabilities. Removing those eagerly avoids provider errors while the
-    callback surfaces a one-time warning to the caller.
+    Currently handles:
+    - GPT-5: only honours temperature=1 and does not return log probabilities.
+    - DashScope Qwen: enforces constraints for logprobs / top_logprobs
     """
+    if model_name.startswith("gpt-5"):
+        _apply_gpt5_filters(params, already_warned, warn)
+        return
 
-    if not model_name.startswith("gpt-5"):
+    if model_name.startswith("dashscope/"):
+        _apply_qwen_dashscope_filters(params, already_warned, warn)
         return
 
+
+def _apply_gpt5_filters(
+    params: Dict[str, Any],
+    already_warned: Set[str],
+    warn: Callable[[str, Any], None],
+) -> None:
+    """Apply GPT-5 specific parameter filters.
+
+    Only honours temperature=1 and does not return log probabilities.
+    Removing those eagerly avoids provider errors while the callback surfaces a
+    one-time warning to the caller.
+    """
+
     unsupported: list[tuple[str, Any]] = []
 
     if "temperature" in params:
@@ -61,7 +78,62 @@ def apply_model_specific_filters(
         if param in params:
             unsupported.append((param, params[param]))
 
-    for param, value in unsupported:
+    _drop_unsupported_params_with_warning(
+        params,
+        unsupported,
+        already_warned,
+        warn,
+    )
+
+
+def _apply_qwen_dashscope_filters(
+    params: Dict[str, Any],
+    already_warned: Set[str],
+    warn: Callable[[str, Any], None],
+) -> None:
+    """Apply Qwen/DashScope specific parameter filters.
+
+    top_logprobs is only meaningful if logprobs is true and must be an int
+    in [0, 5]. When logprobs is false, drops top_logprobs; when logprobs is
+    true, clamps top_logprobs into [0, 5].
+    """
+
+    unsupported: list[tuple[str, Any]] = []
+
+    logprobs_value = params.get("logprobs")
+    if not logprobs_value:
+        if "top_logprobs" in params:
+            unsupported.append(("top_logprobs", params["top_logprobs"]))
+    else:
+        if "top_logprobs" in params:
+            raw_top_logprobs = params["top_logprobs"]
+            try:
+                top_logprobs = int(raw_top_logprobs)
+            except (TypeError, ValueError):
+                unsupported.append(("top_logprobs", raw_top_logprobs))
+            else:
+                if top_logprobs < 0:
+                    top_logprobs = 0
+                elif top_logprobs > 5:
+                    top_logprobs = 5
+                params["top_logprobs"] = top_logprobs
+
+    _drop_unsupported_params_with_warning(
+        params,
+        unsupported,
+        already_warned,
+        warn,
+    )
+
+
+def _drop_unsupported_params_with_warning(
+    params: Dict[str, Any],
+    unsupported_params: list[tuple[str, Any]],
+    already_warned: Set[str],
+    warn: Callable[[str, Any], None],
+) -> None:
+    """Remove unsupported params and emit warnings once per param name."""
+    for param, value in unsupported_params:
         params.pop(param, None)
         if param in already_warned:
             continue

opik/evaluation/report.py

@@ -1,10 +1,11 @@
 from collections import defaultdict
-from typing import Dict, List, Tuple
+from typing import Dict, List, Optional, Tuple
 
 from rich import align, console, panel, table, text
 
 
 from . import test_result, evaluation_result
+from .metrics import score_result
 
 
 def _format_time(seconds: float) -> str:
@@ -41,7 +42,10 @@ def _compute_average_scores(
 
 
 def display_experiment_results(
-    dataset_name: str, total_time: float, test_results: List[test_result.TestResult]
+    dataset_name: str,
+    total_time: float,
+    test_results: List[test_result.TestResult],
+    experiment_scores: Optional[List[score_result.ScoreResult]] = None,
 ) -> None:
     average_scores, failed_scores = _compute_average_scores(test_results)
     nb_items = len(test_results)
@@ -62,6 +66,14 @@ def display_experiment_results(
             score_strings += text.Text(f" - {failed_scores[name]} failed", style="red")
         score_strings += text.Text("\n")
 
+    # Add experiment scores if available
+    if experiment_scores:
+        for score in experiment_scores:
+            score_strings += text.Text(
+                f"{score.name}: {score.value:.4f}", style="green bold"
+            )
+            score_strings += text.Text("\n")
+
     aligned_test_results = align.Align.left(score_strings)
 
     # Combine table, time text, and test results

opik/evaluation/rest_operations.py

@@ -4,7 +4,7 @@ from typing import List, Optional
 from opik.api_objects import dataset, experiment, opik_client
 from opik.types import FeedbackScoreDict
 from . import test_case
-from .metrics import arguments_helpers, score_result
+from .metrics import score_result
 from .types import ScoringKeyMappingType
 
 LOGGER = logging.getLogger(__name__)
@@ -67,11 +67,7 @@ def get_experiment_test_cases(
                 trace_id=item.trace_id,
                 dataset_item_id=item.dataset_item_id,
                 task_output=item.evaluation_task_output,
-                scoring_inputs=arguments_helpers.create_scoring_inputs(
-                    dataset_item=dataset_item_data,
-                    task_output=item.evaluation_task_output,
-                    scoring_key_mapping=scoring_key_mapping,
-                ),
+                dataset_item_content=dataset_item_data,
             )
         )
 
@@ -98,6 +94,7 @@ def log_test_result_feedback_scores(
         )
         all_trace_scores.append(trace_score)
 
-    client.log_traces_feedback_scores(
-        scores=all_trace_scores, project_name=project_name
-    )
+    if len(all_trace_scores) > 0:
+        client.log_traces_feedback_scores(
+            scores=all_trace_scores, project_name=project_name
+        )

opik/evaluation/test_case.py

@@ -1,4 +1,4 @@
-from typing import Dict, Any
+from typing import Dict, Any, Optional
 import dataclasses
 
 
@@ -6,6 +6,6 @@ import dataclasses
 class TestCase:
     trace_id: str
     dataset_item_id: str
-    scoring_inputs: Dict[str, Any]
     task_output: Dict[str, Any]
     dataset_item_content: Dict[str, Any] = dataclasses.field(default_factory=dict)
+    mapped_scoring_inputs: Optional[Dict[str, Any]] = None

opik/evaluation/types.py

@@ -1,5 +1,13 @@
-from typing import Any, Callable, Dict, Union
+from typing import Any, Callable, Dict, List, Union
+
+from . import test_result
+from .metrics import score_result
 
 LLMTask = Callable[[Dict[str, Any]], Dict[str, Any]]
 
 ScoringKeyMappingType = Dict[str, Union[str, Callable[[Dict[str, Any]], Any]]]
+
+ExperimentScoreFunction = Callable[
+    [List[test_result.TestResult]],
+    Union[score_result.ScoreResult, List[score_result.ScoreResult]],
+]

opik/exceptions.py

@@ -81,6 +81,23 @@ class PromptPlaceholdersDontMatchFormatArguments(OpikException):
         )
 
 
+class PromptTemplateStructureMismatch(OpikException):
+    """Exception raised when attempting to create a prompt version with a different template structure than the existing prompt."""
+
+    def __init__(
+        self, prompt_name: str, existing_structure: str, attempted_structure: str
+    ):
+        self.prompt_name = prompt_name
+        self.existing_structure = existing_structure
+        self.attempted_structure = attempted_structure
+
+    def __str__(self) -> str:
+        return (
+            f"Prompt with name '{self.prompt_name}' already exists and has immutable "
+            f"'{self.existing_structure}' template structure, not '{self.attempted_structure}'. "
+        )
+
+
 class ExperimentNotFound(OpikException):
     pass
 

opik/id_helpers.py

@@ -1,5 +1,7 @@
 from datetime import datetime
 from typing import Optional
+import random
+import string
 import uuid
 import uuid6
 
@@ -12,6 +14,22 @@ def generate_id(timestamp: Optional[datetime] = None) -> str:
     return str(uuid6.uuid7())
 
 
+def generate_random_alphanumeric_string(length: int) -> str:
+    """Generate a random alphanumeric string of the specified length.
+
+    Args:
+        length: The length of the string to generate.
+
+    Returns:
+        A random string containing only alphanumeric characters (a-z, A-Z, 0-9).
+    """
+    if length < 0:
+        raise ValueError("Length must be non-negative")
+
+    characters = string.ascii_letters + string.digits
+    return "".join(random.choice(characters) for _ in range(length))
+
+
 def uuid4_to_uuid7(user_datetime: datetime, user_uuid: str) -> uuid.UUID:
     """Convert a UUID v4 into a UUID v7 following RFC draft specification."""
     # Get Unix timestamp in milliseconds

opik/integrations/adk/helpers.py

@@ -1,3 +1,4 @@
+import logging
 import os
 from typing import (
     Any,
@@ -8,6 +9,8 @@ from google.adk.models import LlmResponse
 import opik.types as opik_types
 import pydantic
 
+LOGGER = logging.getLogger(__name__)
+
 
 def convert_adk_base_model_to_dict(value: pydantic.BaseModel) -> Dict[str, Any]:
     """Most ADK objects are Pydantic Base Models"""
@@ -27,13 +30,19 @@ def get_adk_provider() -> opik_types.LLMProvider:
 
 
 def has_empty_text_part_content(llm_response: LlmResponse) -> bool:
-    if llm_response.content is None or len(llm_response.content.parts) == 0:
-        return True
+    try:
+        if llm_response.content is None:
+            return True
 
-    # to filter out something like this: {"candidates":[{"content":{"parts":[{"text":""}],"role":"model"}}],...}}
-    if len(llm_response.content.parts) == 1:
-        part = llm_response.content.parts[0]
-        if part.text is not None and len(part.text) == 0:
+        if not llm_response.content.parts:
             return True
 
-    return False
+        # to filter out something like this: {"candidates":[{"content":{"parts":[{"text":""}],"role":"model"}}],...}}
+        if len(llm_response.content.parts) == 1:
+            part = llm_response.content.parts[0]
+            if part.text is not None and len(part.text) == 0:
+                return True
+        return False
+    except Exception as e:
+        LOGGER.warning(f"Exception in has_empty_text_part_content {e}", exc_info=True)
+        return True

opik/integrations/adk/legacy_opik_tracer.py

@@ -8,6 +8,7 @@ from google.adk import models
 from google.adk.tools import base_tool
 from google.adk.tools import tool_context
 
+import opik
 from opik import context_storage
 from opik.decorator import arguments_helpers, span_creation_handler
 from opik.api_objects import opik_client, span, trace
@@ -77,7 +78,8 @@ class LegacyOpikTracer:
         trace_data = self._context_storage.pop_trace_data()
         assert trace_data is not None
         trace_data.init_end_time()
-        self._opik_client.trace(**trace_data.as_parameters)
+        if opik.is_tracing_active():
+            self._opik_client.trace(**trace_data.as_parameters)
 
     def _end_current_span(
         self,
@@ -85,20 +87,21 @@ class LegacyOpikTracer:
         span_data = self._context_storage.pop_span_data()
         assert span_data is not None
         span_data.init_end_time()
-        self._opik_client.span(**span_data.as_parameters)
+        if opik.is_tracing_active():
+            self._opik_client.span(**span_data.as_parameters)
 
     def _start_span(self, span_data: span.SpanData) -> None:
         self._context_storage.add_span_data(span_data)
         self._opik_created_spans.add(span_data.id)
 
-        if self._opik_client.config.log_start_trace_span:
+        if self._opik_client.config.log_start_trace_span and opik.is_tracing_active():
             self._opik_client.span(**span_data.as_start_parameters)
 
     def _start_trace(self, trace_data: trace.TraceData) -> None:
         self._context_storage.set_trace_data(trace_data)
         self._current_trace_created_by_opik_tracer.set(trace_data.id)
 
-        if self._opik_client.config.log_start_trace_span:
+        if self._opik_client.config.log_start_trace_span and opik.is_tracing_active():
             self._opik_client.trace(**trace_data.as_start_parameters)
 
     def _set_current_context_data(self, value: SpanOrTraceData) -> None:

opik/integrations/adk/opik_tracer.py

@@ -7,6 +7,7 @@ from google.adk import models
 from google.adk.tools import base_tool
 from google.adk.tools import tool_context
 
+import opik
 from opik import context_storage
 from opik.api_objects import opik_client, span, trace
 from opik.types import DistributedTraceHeadersDict
@@ -253,7 +254,8 @@ class OpikTracer:
             current_span.init_end_time()
             # We close this span manually because otherwise ADK will close it too late,
             # and it will also add tool spans inside of it, which we want to avoid.
-            self._opik_client.span(**current_span.as_parameters)
+            if opik.is_tracing_active():
+                self._opik_client.span(**current_span.as_parameters)
             self._last_model_output = output
 
         except Exception as e:

opik/integrations/adk/patchers/adk_otel_tracer/opik_adk_otel_tracer.py

@@ -2,6 +2,7 @@ import logging
 from typing import Iterator, Optional, Tuple
 
 import opentelemetry.trace
+import opik
 import opik.context_storage
 from opik.api_objects import trace, span
 from opik.decorator import (
@@ -100,7 +101,8 @@ class OpikADKOtelTracer(opentelemetry.trace.NoOpTracer):
             # so we manually finalize it here to avoid incorrect span nesting.
             opik.context_storage.pop_span_data(ensure_id=current_span_data.id)
             current_span_data.init_end_time()
-            self.opik_client.span(**current_span_data.as_parameters)
+            if opik.is_tracing_active():
+                self.opik_client.span(**current_span_data.as_parameters)
             current_span_data = opik.context_storage.top_span_data()
 
         try:
@@ -145,7 +147,8 @@ class OpikADKOtelTracer(opentelemetry.trace.NoOpTracer):
         trace_data = opik.context_storage.pop_trace_data(ensure_id=trace_id)
         if trace_data is not None:
             trace_data.init_end_time()
-            self.opik_client.trace(**trace_data.as_parameters)
+            if opik.is_tracing_active():
+                self.opik_client.trace(**trace_data.as_parameters)
 
     def _ensure_span_is_finalized(self, span_id: str) -> None:
         opik.context_storage.trim_span_data_stack_to_certain_span(span_id)
@@ -153,7 +156,8 @@ class OpikADKOtelTracer(opentelemetry.trace.NoOpTracer):
         span_data = opik.context_storage.pop_span_data(ensure_id=span_id)
         if span_data is not None:
             span_data.init_end_time()
-            self.opik_client.span(**span_data.as_parameters)
+            if opik.is_tracing_active():
+                self.opik_client.span(**span_data.as_parameters)
 
 
 def _prepare_trace_and_span_to_be_finalized(

opik/integrations/dspy/callback.py

@@ -4,11 +4,8 @@ import logging
 import dspy
 from dspy.utils import callback as dspy_callback
 
-import opik.types as types
-import opik.opik_context as opik_context
-import opik.context_storage as context_storage
+from opik import context_storage, opik_context, tracing_runtime_config, types
 from opik.api_objects import helpers, span, trace, opik_client
-import opik.decorator.tracing_runtime_config as tracing_runtime_config
 from opik.decorator import error_info_collector
 
 from .graph import build_mermaid_graph_from_module

opik/integrations/haystack/opik_connector.py

@@ -4,8 +4,8 @@ from typing import Any, Dict, Optional
 import haystack
 from haystack import tracing
 
-import opik.api_objects.opik_client as opik_client
-import opik.decorator.tracing_runtime_config as tracing_runtime_config
+from opik import tracing_runtime_config
+from opik.api_objects import opik_client
 from . import opik_tracer
 
 LOGGER = logging.getLogger(__name__)

opik/integrations/haystack/opik_tracer.py

@@ -5,10 +5,8 @@ from typing import Any, Dict, Iterator, List, Optional, Union
 
 from haystack import tracing
 
-import opik.url_helpers as url_helpers
-import opik.decorator.tracing_runtime_config as tracing_runtime_config
-import opik.decorator.span_creation_handler as span_creation_handler
-import opik.decorator.arguments_helpers as arguments_helpers
+from opik import tracing_runtime_config, url_helpers
+from opik.decorator import arguments_helpers, span_creation_handler
 from opik.api_objects import opik_client
 from opik.api_objects import span as opik_span
 from opik.api_objects import trace as opik_trace

opik/integrations/langchain/opik_tracer.py

@@ -19,8 +19,7 @@ from langchain_core import language_models
 from langchain_core.tracers import BaseTracer
 from langchain_core.tracers.schemas import Run
 
-import opik.dict_utils as dict_utils
-import opik.llm_usage as llm_usage
+from opik import context_storage, dict_utils, llm_usage, tracing_runtime_config
 from opik.api_objects import span, trace
 from opik.types import DistributedTraceHeadersDict, ErrorInfoDict
 from opik.validation import parameters_validator
@@ -32,8 +31,6 @@ from . import (
 )
 
 from ...api_objects import helpers, opik_client
-import opik.context_storage as context_storage
-import opik.decorator.tracing_runtime_config as tracing_runtime_config
 
 if TYPE_CHECKING:
     from langchain_core.runnables.graph import Graph

opik/integrations/llama_index/callback.py

@@ -6,10 +6,8 @@ import uuid
 from llama_index.core.callbacks import schema as llama_index_schema
 from llama_index.core.callbacks import base_handler
 
-import opik.context_storage as context_storage
-import opik.decorator.tracing_runtime_config as tracing_runtime_config
-import opik.decorator.arguments_helpers as arguments_helpers
-import opik.decorator.span_creation_handler as span_creation_handler
+from opik import context_storage, tracing_runtime_config
+from opik.decorator import arguments_helpers, span_creation_handler
 from opik.api_objects import opik_client, span, trace
 
 from . import event_parsing_utils

opik/integrations/openai/agents/opik_tracing_processor.py

@@ -3,12 +3,11 @@ from agents import tracing
 
 import logging
 
+from opik import context_storage, tracing_runtime_config
 from opik.api_objects.span import span_data
 from opik.api_objects.trace import trace_data
 from opik.api_objects import opik_client
 from opik.decorator import span_creation_handler, arguments_helpers
-import opik.decorator.tracing_runtime_config as tracing_runtime_config
-import opik.context_storage as context_storage
 
 from . import span_data_parsers
 

opik/integrations/openai/opik_tracker.py

@@ -18,7 +18,7 @@ def track_openai(
     """Adds Opik tracking wrappers to an OpenAI client.
 
     The client is always patched; however every wrapped call checks
-    `opik.decorator.tracing_runtime_config.is_tracing_active()` before emitting
+    `opik.is_tracing_active()` before emitting
     any telemetry. If tracing is disabled at call time, the wrapped function
     executes normally but no span/trace is sent.
 

opik/opik_context.py

@@ -1,7 +1,7 @@
 import contextlib
-from typing import Any, Dict, List, Optional, Union, Iterator
+from typing import Any, Dict, List, Optional, Iterator, Union
 
-import opik.llm_usage as llm_usage
+from opik import llm_usage
 from opik.api_objects import span, trace, opik_client, prompt
 from opik.api_objects.attachment import Attachment
 from opik.types import (
@@ -11,7 +11,7 @@ from opik.types import (
     ErrorInfoDict,
 )
 
-import opik.decorator.tracing_runtime_config as tracing_runtime_config
+from opik import tracing_runtime_config
 
 from . import context_storage, exceptions
 from .decorator import error_info_collector
@@ -68,7 +68,7 @@ def update_current_span(
     total_cost: Optional[float] = None,
     attachments: Optional[List[Attachment]] = None,
     error_info: Optional[ErrorInfoDict] = None,
-    prompts: Optional[List[prompt.Prompt]] = None,
+    prompts: Optional[List[prompt.BasePrompt]] = None,
 ) -> None:
     """
     Update the current span with the provided parameters. This method is usually called within a tracked function.
@@ -97,7 +97,7 @@ def update_current_span(
         return
 
     if prompts is not None:
-        prompts = [prompt.prompt.to_info_dict(p) for p in prompts]
+        prompts = [p.__internal_api__to_info_dict__() for p in prompts]
 
     new_params = {
         "name": name,
@@ -130,7 +130,7 @@ def update_current_trace(
     feedback_scores: Optional[List[FeedbackScoreDict]] = None,
     thread_id: Optional[str] = None,
     attachments: Optional[List[Attachment]] = None,
-    prompts: Optional[List[prompt.Prompt]] = None,
+    prompts: Optional[List[prompt.BasePrompt]] = None,
 ) -> None:
     """
     Update the current trace with the provided parameters. This method is usually called within a tracked function.
@@ -151,7 +151,7 @@ def update_current_trace(
         return
 
     if prompts is not None:
-        prompts = [prompt.prompt.to_info_dict(p) for p in prompts]
+        prompts = [p.__internal_api__to_info_dict__() for p in prompts]
 
     new_params = {
         "name": name,